{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import PyPDF2 as pypdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of Pages: 178\n"
     ]
    }
   ],
   "source": [
    "# Creating a pdf file object.\n",
    "pdf = open(\"companion.pdf\", \"rb\")\n",
    " \n",
    "# Creating pdf reader object.\n",
    "pdf_reader = PyPDF2.PdfFileReader(pdf)\n",
    " \n",
    "# Checking total number of pages in a pdf file.\n",
    "print(\"Total number of Pages:\", pdf_reader.numPages)\n",
    " \n",
    "# # Creating a page object.\n",
    "# page = pdf_reader.getPage(200)\n",
    " \n",
    "# # Extract data from a specific page number.\n",
    "# print(page.extractText())\n",
    " \n",
    "# # Closing the object.\n",
    "# pdf.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Chapter2\n",
      "Linearfunctions\n",
      "2.1Linearfunctions\n",
      "FunctionsinJulia.\n",
      "Juliaprovidesseveralmethodsforfunctions.Asim-\n",
      "plefunctiongivenbyanexpressionsuchas\n",
      "f\n",
      "(\n",
      "x\n",
      ")=\n",
      "x\n",
      "1\n",
      "+\n",
      "x\n",
      "2\n",
      "\n",
      "x\n",
      "2\n",
      "4\n",
      "canbein\n",
      "asingleline.\n",
      "julia>\n",
      "f(x)=x[1]+x[2]-x[4]^2\n",
      "f(genericfunctionwith1method)\n",
      "julia>\n",
      "f([-1,0,1,2])\n",
      "-5\n",
      "Sincethefunctionreferstothesecond,andfourthelementsofthe\n",
      "argument\n",
      "x\n",
      ",thesehavetobewhenyoucallorevaluate\n",
      "f(x)\n",
      ";you'llgetan\n",
      "errorif,forexample,\n",
      "x\n",
      "hasdimension3orisascalar.\n",
      "Superposition.\n",
      "Suppose\n",
      "a\n",
      "isan\n",
      "n\n",
      "-vector.Thefunction\n",
      "f\n",
      "(\n",
      "x\n",
      ")=\n",
      "a\n",
      "T\n",
      "x\n",
      "islinear,\n",
      "whichmeansthatforany\n",
      "n\n",
      "-vectors\n",
      "x\n",
      "and\n",
      "y\n",
      ",andanyscalars\n",
      "\n",
      "and\n",
      "\n",
      ",thesuper-\n",
      "positionequality\n",
      "f\n",
      "(\n",
      "x\n",
      "+\n",
      "y\n",
      ")=\n",
      "f\n",
      "(\n",
      "x\n",
      ")+\n",
      "f\n",
      "(\n",
      "y\n",
      ")\n",
      "holds.Superpositionsaysthatevaluating\n",
      "f\n",
      "atalinearcombinationoftwovectors\n",
      "isthesameformingthelinearcombinationof\n",
      "f\n",
      "evaluatedatthetwovectors.\n",
      "Let'stheinnerproductfunction\n",
      "f\n",
      "foraspvalueof\n",
      "a\n",
      ",andthen\n",
      "verifysuperpositioninJuliaforspvaluesof\n",
      "x\n",
      ",\n",
      "y\n",
      ",\n",
      "\n",
      ",and\n",
      "\n",
      ".(Thischeckdoes\n",
      "notshowthatthefunctionislinear.Itsimplychecksthatsuperpositionholdsfor\n",
      "thesespvalues.)\n",
      "julia>\n",
      "a=[-2,0,1,-3];\n",
      "julia>\n",
      "f(x)=a\n",
      "'\n",
      "*x#Innerproductfunction\n",
      "f(genericfunctionwith1method)\n",
      "julia>\n",
      "x=[2,2,-1,1];y=[0,1,-1,0];\n",
      "\n"
     ]
    }
   ],
   "source": [
    "page = pdf_reader.getPage(28)\n",
    "print(page.extractText())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'/Type': '/Page',\n",
       " '/Contents': {'/Filter': '/FlateDecode'},\n",
       " '/Resources': {'/ColorSpace': {'/pgfprgb': ['/Pattern', '/DeviceRGB']},\n",
       "  '/Pattern': {},\n",
       "  '/ExtGState': {'/pgf@ca1.0': {'/ca': 1}},\n",
       "  '/Font': {'/F17': {'/Type': '/Font',\n",
       "    '/Subtype': '/Type1',\n",
       "    '/BaseFont': '/BHYOKA+CMSSBX10',\n",
       "    '/FontDescriptor': {'/Type': '/FontDescriptor',\n",
       "     '/FontName': '/BHYOKA+CMSSBX10',\n",
       "     '/Flags': 4,\n",
       "     '/FontBBox': [-71, -250, 1099, 780],\n",
       "     '/Ascent': 694,\n",
       "     '/CapHeight': 694,\n",
       "     '/Descent': -194,\n",
       "     '/ItalicAngle': 0,\n",
       "     '/StemV': 136,\n",
       "     '/XHeight': 458,\n",
       "     '/CharSet': '/A/B/C/D/E/F/G/H/I/J/K/L/M/N/O/P/Q/R/S/T/U/V/Z/a/b/c/comma/d/e/eight/endash/f/ff/ffi/fi/five/fl/four/g/h/hyphen/i/j/k/l/m/n/nine/o/one/p/period/q/r/s/seven/six/t/three/two/u/v/w/x/y/z/zero',\n",
       "     '/FontFile': {'/Length1': 2387,\n",
       "      '/Length2': 15242,\n",
       "      '/Length3': 0,\n",
       "      '/Filter': '/FlateDecode'}},\n",
       "    '/FirstChar': 11,\n",
       "    '/LastChar': 123,\n",
       "    '/Widths': [641.7,\n",
       "     586.1,\n",
       "     586.1,\n",
       "     891.7,\n",
       "     891.7,\n",
       "     255.6,\n",
       "     286.1,\n",
       "     550,\n",
       "     550,\n",
       "     550,\n",
       "     550,\n",
       "     550,\n",
       "     733.3,\n",
       "     488.9,\n",
       "     565.3,\n",
       "     794.4,\n",
       "     855.6,\n",
       "     550,\n",
       "     947.2,\n",
       "     1069.5,\n",
       "     855.6,\n",
       "     255.6,\n",
       "     366.7,\n",
       "     558.3,\n",
       "     916.7,\n",
       "     550,\n",
       "     1029.1,\n",
       "     830.6,\n",
       "     305.6,\n",
       "     427.8,\n",
       "     427.8,\n",
       "     550,\n",
       "     855.6,\n",
       "     305.6,\n",
       "     366.7,\n",
       "     305.6,\n",
       "     550,\n",
       "     550,\n",
       "     550,\n",
       "     550,\n",
       "     550,\n",
       "     550,\n",
       "     550,\n",
       "     550,\n",
       "     550,\n",
       "     550,\n",
       "     550,\n",
       "     305.6,\n",
       "     305.6,\n",
       "     366.7,\n",
       "     855.6,\n",
       "     519.4,\n",
       "     519.4,\n",
       "     733.3,\n",
       "     733.3,\n",
       "     733.3,\n",
       "     702.8,\n",
       "     794.4,\n",
       "     641.7,\n",
       "     611.1,\n",
       "     733.3,\n",
       "     794.4,\n",
       "     330.6,\n",
       "     519.4,\n",
       "     763.9,\n",
       "     580.6,\n",
       "     977.8,\n",
       "     794.4,\n",
       "     794.4,\n",
       "     702.8,\n",
       "     794.4,\n",
       "     702.8,\n",
       "     611.1,\n",
       "     733.3,\n",
       "     763.9,\n",
       "     733.3,\n",
       "     1038.9,\n",
       "     733.3,\n",
       "     733.3,\n",
       "     672.2,\n",
       "     343.1,\n",
       "     558.3,\n",
       "     343.1,\n",
       "     550,\n",
       "     305.6,\n",
       "     305.6,\n",
       "     525,\n",
       "     561.1,\n",
       "     488.9,\n",
       "     561.1,\n",
       "     511.1,\n",
       "     336.1,\n",
       "     550,\n",
       "     561.1,\n",
       "     255.6,\n",
       "     286.1,\n",
       "     530.6,\n",
       "     255.6,\n",
       "     866.7,\n",
       "     561.1,\n",
       "     550,\n",
       "     561.1,\n",
       "     561.1,\n",
       "     372.2,\n",
       "     421.7,\n",
       "     404.2,\n",
       "     561.1,\n",
       "     500,\n",
       "     744.4,\n",
       "     500,\n",
       "     500,\n",
       "     476.4,\n",
       "     550]},\n",
       "   '/F8': {'/Type': '/Font',\n",
       "    '/Subtype': '/Type1',\n",
       "    '/BaseFont': '/WUVEBD+CMR10',\n",
       "    '/FontDescriptor': {'/Type': '/FontDescriptor',\n",
       "     '/FontName': '/WUVEBD+CMR10',\n",
       "     '/Flags': 4,\n",
       "     '/FontBBox': [-40, -250, 1009, 750],\n",
       "     '/Ascent': 694,\n",
       "     '/CapHeight': 683,\n",
       "     '/Descent': -194,\n",
       "     '/ItalicAngle': 0,\n",
       "     '/StemV': 69,\n",
       "     '/XHeight': 431,\n",
       "     '/CharSet': '/A/B/C/D/E/F/G/H/I/J/K/L/M/N/O/P/Q/R/S/T/Theta/U/V/W/X/Y/Z/a/b/bracketleft/bracketright/c/circumflex/colon/comma/d/dieresis/dotlessi/e/eight/endash/equal/exclam/f/ff/ffi/fi/five/fl/four/g/h/hyphen/i/j/k/l/m/n/nine/o/one/p/parenleft/parenright/percent/period/plus/q/quotedblleft/quotedblright/quoteleft/quoteright/r/s/semicolon/seven/six/slash/t/three/tilde/two/u/v/w/x/y/z/zero',\n",
       "     '/FontFile': {'/Length1': 2831,\n",
       "      '/Length2': 24201,\n",
       "      '/Length3': 0,\n",
       "      '/Filter': '/FlateDecode'}},\n",
       "    '/FirstChar': 2,\n",
       "    '/LastChar': 127,\n",
       "    '/Widths': [777.8,\n",
       "     694.4,\n",
       "     666.7,\n",
       "     750,\n",
       "     722.2,\n",
       "     777.8,\n",
       "     722.2,\n",
       "     777.8,\n",
       "     722.2,\n",
       "     583.3,\n",
       "     555.6,\n",
       "     555.6,\n",
       "     833.3,\n",
       "     833.3,\n",
       "     277.8,\n",
       "     305.6,\n",
       "     500,\n",
       "     500,\n",
       "     500,\n",
       "     500,\n",
       "     500,\n",
       "     750,\n",
       "     444.4,\n",
       "     500,\n",
       "     722.2,\n",
       "     777.8,\n",
       "     500,\n",
       "     902.8,\n",
       "     1013.9,\n",
       "     777.8,\n",
       "     277.8,\n",
       "     277.8,\n",
       "     500,\n",
       "     833.3,\n",
       "     500,\n",
       "     833.3,\n",
       "     777.8,\n",
       "     277.8,\n",
       "     388.9,\n",
       "     388.9,\n",
       "     500,\n",
       "     777.8,\n",
       "     277.8,\n",
       "     333.3,\n",
       "     277.8,\n",
       "     500,\n",
       "     500,\n",
       "     500,\n",
       "     500,\n",
       "     500,\n",
       "     500,\n",
       "     500,\n",
       "     500,\n",
       "     500,\n",
       "     500,\n",
       "     500,\n",
       "     277.8,\n",
       "     277.8,\n",
       "     277.8,\n",
       "     777.8,\n",
       "     472.2,\n",
       "     472.2,\n",
       "     777.8,\n",
       "     750,\n",
       "     708.3,\n",
       "     722.2,\n",
       "     763.9,\n",
       "     680.6,\n",
       "     652.8,\n",
       "     784.7,\n",
       "     750,\n",
       "     361.1,\n",
       "     513.9,\n",
       "     777.8,\n",
       "     625,\n",
       "     916.7,\n",
       "     750,\n",
       "     777.8,\n",
       "     680.6,\n",
       "     777.8,\n",
       "     736.1,\n",
       "     555.6,\n",
       "     722.2,\n",
       "     750,\n",
       "     750,\n",
       "     1027.8,\n",
       "     750,\n",
       "     750,\n",
       "     611.1,\n",
       "     277.8,\n",
       "     500,\n",
       "     277.8,\n",
       "     500,\n",
       "     277.8,\n",
       "     277.8,\n",
       "     500,\n",
       "     555.6,\n",
       "     444.4,\n",
       "     555.6,\n",
       "     444.4,\n",
       "     305.6,\n",
       "     500,\n",
       "     555.6,\n",
       "     277.8,\n",
       "     305.6,\n",
       "     527.8,\n",
       "     277.8,\n",
       "     833.3,\n",
       "     555.6,\n",
       "     500,\n",
       "     555.6,\n",
       "     527.8,\n",
       "     391.7,\n",
       "     394.4,\n",
       "     388.9,\n",
       "     555.6,\n",
       "     527.8,\n",
       "     722.2,\n",
       "     527.8,\n",
       "     527.8,\n",
       "     444.4,\n",
       "     500,\n",
       "     1000,\n",
       "     500,\n",
       "     500,\n",
       "     500]},\n",
       "   '/F11': {'/Type': '/Font',\n",
       "    '/Subtype': '/Type1',\n",
       "    '/BaseFont': '/UHRIBG+CMMI10',\n",
       "    '/FontDescriptor': {'/Type': '/FontDescriptor',\n",
       "     '/FontName': '/UHRIBG+CMMI10',\n",
       "     '/Flags': 4,\n",
       "     '/FontBBox': [-32, -250, 1048, 750],\n",
       "     '/Ascent': 694,\n",
       "     '/CapHeight': 683,\n",
       "     '/Descent': -194,\n",
       "     '/ItalicAngle': -14,\n",
       "     '/StemV': 72,\n",
       "     '/XHeight': 431,\n",
       "     '/CharSet': '/A/B/C/D/E/F/G/H/I/J/K/M/N/Q/R/T/X/Y/Z/a/alpha/b/beta/c/comma/d/e/f/g/greater/i/j/k/l/lambda/m/mu/n/omega/p/partialdiff/period/phi/pi/q/r/rho/s/slash/t/theta/u/v/w/x/y/z',\n",
       "     '/FontFile': {'/Length1': 2253,\n",
       "      '/Length2': 18092,\n",
       "      '/Length3': 0,\n",
       "      '/Filter': '/FlateDecode'}},\n",
       "    '/FirstChar': 11,\n",
       "    '/LastChar': 122,\n",
       "    '/Widths': [639.7,\n",
       "     565.6,\n",
       "     517.7,\n",
       "     444.4,\n",
       "     405.9,\n",
       "     437.5,\n",
       "     496.5,\n",
       "     469.4,\n",
       "     353.9,\n",
       "     576.2,\n",
       "     583.3,\n",
       "     602.6,\n",
       "     494,\n",
       "     437.5,\n",
       "     570,\n",
       "     517,\n",
       "     571.4,\n",
       "     437.2,\n",
       "     540.3,\n",
       "     595.8,\n",
       "     625.7,\n",
       "     651.4,\n",
       "     622.5,\n",
       "     466.3,\n",
       "     591.4,\n",
       "     828.1,\n",
       "     517,\n",
       "     362.8,\n",
       "     654.2,\n",
       "     1000,\n",
       "     1000,\n",
       "     1000,\n",
       "     1000,\n",
       "     277.8,\n",
       "     277.8,\n",
       "     500,\n",
       "     500,\n",
       "     500,\n",
       "     500,\n",
       "     500,\n",
       "     500,\n",
       "     500,\n",
       "     500,\n",
       "     500,\n",
       "     500,\n",
       "     500,\n",
       "     500,\n",
       "     277.8,\n",
       "     277.8,\n",
       "     777.8,\n",
       "     500,\n",
       "     777.8,\n",
       "     500,\n",
       "     530.9,\n",
       "     750,\n",
       "     758.5,\n",
       "     714.7,\n",
       "     827.9,\n",
       "     738.2,\n",
       "     643.1,\n",
       "     786.3,\n",
       "     831.3,\n",
       "     439.6,\n",
       "     554.5,\n",
       "     849.3,\n",
       "     680.6,\n",
       "     970.1,\n",
       "     803.5,\n",
       "     762.8,\n",
       "     642,\n",
       "     790.6,\n",
       "     759.3,\n",
       "     613.2,\n",
       "     584.4,\n",
       "     682.8,\n",
       "     583.3,\n",
       "     944.4,\n",
       "     828.5,\n",
       "     580.6,\n",
       "     682.6,\n",
       "     388.9,\n",
       "     388.9,\n",
       "     388.9,\n",
       "     1000,\n",
       "     1000,\n",
       "     416.7,\n",
       "     528.6,\n",
       "     429.2,\n",
       "     432.8,\n",
       "     520.5,\n",
       "     465.6,\n",
       "     489.6,\n",
       "     477,\n",
       "     576.2,\n",
       "     344.5,\n",
       "     411.8,\n",
       "     520.6,\n",
       "     298.4,\n",
       "     878,\n",
       "     600.2,\n",
       "     484.7,\n",
       "     503.1,\n",
       "     446.4,\n",
       "     451.2,\n",
       "     468.8,\n",
       "     361.1,\n",
       "     572.5,\n",
       "     484.7,\n",
       "     715.9,\n",
       "     571.5,\n",
       "     490.3,\n",
       "     465.1]},\n",
       "   '/F7': {'/Type': '/Font',\n",
       "    '/Subtype': '/Type1',\n",
       "    '/BaseFont': '/JDDZOM+CMR7',\n",
       "    '/FontDescriptor': {'/Type': '/FontDescriptor',\n",
       "     '/FontName': '/JDDZOM+CMR7',\n",
       "     '/Flags': 4,\n",
       "     '/FontBBox': [-27, -250, 1122, 750],\n",
       "     '/Ascent': 694,\n",
       "     '/CapHeight': 683,\n",
       "     '/Descent': -194,\n",
       "     '/ItalicAngle': 0,\n",
       "     '/StemV': 79,\n",
       "     '/XHeight': 431,\n",
       "     '/CharSet': '/a/c/colon/d/e/eight/equal/five/four/l/m/n/nine/o/one/parenleft/parenright/plus/s/seven/six/t/three/two/u/x/zero',\n",
       "     '/FontFile': {'/Length1': 1787,\n",
       "      '/Length2': 10345,\n",
       "      '/Length3': 0,\n",
       "      '/Filter': '/FlateDecode'}},\n",
       "    '/FirstChar': 40,\n",
       "    '/LastChar': 120,\n",
       "    '/Widths': [446.4,\n",
       "     446.4,\n",
       "     569.5,\n",
       "     877,\n",
       "     323.4,\n",
       "     384.9,\n",
       "     323.4,\n",
       "     569.5,\n",
       "     569.5,\n",
       "     569.5,\n",
       "     569.5,\n",
       "     569.5,\n",
       "     569.5,\n",
       "     569.5,\n",
       "     569.5,\n",
       "     569.5,\n",
       "     569.5,\n",
       "     569.5,\n",
       "     323.4,\n",
       "     323.4,\n",
       "     323.4,\n",
       "     877,\n",
       "     538.7,\n",
       "     538.7,\n",
       "     877,\n",
       "     843.3,\n",
       "     798.6,\n",
       "     815.5,\n",
       "     860.1,\n",
       "     767.9,\n",
       "     737.1,\n",
       "     883.9,\n",
       "     843.3,\n",
       "     412.7,\n",
       "     583.3,\n",
       "     874,\n",
       "     706.4,\n",
       "     1027.8,\n",
       "     843.3,\n",
       "     877,\n",
       "     767.9,\n",
       "     877,\n",
       "     829.4,\n",
       "     631,\n",
       "     815.5,\n",
       "     843.3,\n",
       "     843.3,\n",
       "     1150.8,\n",
       "     843.3,\n",
       "     843.3,\n",
       "     692.5,\n",
       "     323.4,\n",
       "     569.5,\n",
       "     323.4,\n",
       "     569.5,\n",
       "     323.4,\n",
       "     323.4,\n",
       "     569.5,\n",
       "     631,\n",
       "     507.9,\n",
       "     631,\n",
       "     507.9,\n",
       "     354.2,\n",
       "     569.5,\n",
       "     631,\n",
       "     323.4,\n",
       "     354.2,\n",
       "     600.2,\n",
       "     323.4,\n",
       "     938.5,\n",
       "     631,\n",
       "     569.5,\n",
       "     631,\n",
       "     600.2,\n",
       "     446.4,\n",
       "     452.6,\n",
       "     446.4,\n",
       "     631,\n",
       "     600.2,\n",
       "     815.5,\n",
       "     600.2]},\n",
       "   '/F14': {'/Type': '/Font',\n",
       "    '/Subtype': '/Type1',\n",
       "    '/BaseFont': '/OIMRRQ+CMSY10',\n",
       "    '/FontDescriptor': {'/Type': '/FontDescriptor',\n",
       "     '/FontName': '/OIMRRQ+CMSY10',\n",
       "     '/Flags': 4,\n",
       "     '/FontBBox': [-29, -960, 1116, 775],\n",
       "     '/Ascent': 750,\n",
       "     '/CapHeight': 683,\n",
       "     '/Descent': -194,\n",
       "     '/ItalicAngle': -14,\n",
       "     '/StemV': 40,\n",
       "     '/XHeight': 431,\n",
       "     '/CharSet': '/arrowright/bardbl/braceleft/braceright/element/lessequal/minus/multiply/nabla/negationslash/periodcentered/plusminus/radical',\n",
       "     '/FontFile': {'/Length1': 1644,\n",
       "      '/Length2': 7654,\n",
       "      '/Length3': 0,\n",
       "      '/Filter': '/FlateDecode'}},\n",
       "    '/FirstChar': 0,\n",
       "    '/LastChar': 114,\n",
       "    '/Widths': [777.8,\n",
       "     277.8,\n",
       "     777.8,\n",
       "     500,\n",
       "     777.8,\n",
       "     500,\n",
       "     777.8,\n",
       "     777.8,\n",
       "     777.8,\n",
       "     777.8,\n",
       "     777.8,\n",
       "     777.8,\n",
       "     777.8,\n",
       "     1000,\n",
       "     500,\n",
       "     500,\n",
       "     777.8,\n",
       "     777.8,\n",
       "     777.8,\n",
       "     777.8,\n",
       "     777.8,\n",
       "     777.8,\n",
       "     777.8,\n",
       "     777.8,\n",
       "     777.8,\n",
       "     777.8,\n",
       "     777.8,\n",
       "     777.8,\n",
       "     1000,\n",
       "     1000,\n",
       "     777.8,\n",
       "     777.8,\n",
       "     1000,\n",
       "     1000,\n",
       "     500,\n",
       "     500,\n",
       "     1000,\n",
       "     1000,\n",
       "     1000,\n",
       "     777.8,\n",
       "     1000,\n",
       "     1000,\n",
       "     611.1,\n",
       "     611.1,\n",
       "     1000,\n",
       "     1000,\n",
       "     1000,\n",
       "     777.8,\n",
       "     275,\n",
       "     1000,\n",
       "     666.7,\n",
       "     666.7,\n",
       "     888.9,\n",
       "     888.9,\n",
       "     0,\n",
       "     0,\n",
       "     555.6,\n",
       "     555.6,\n",
       "     666.7,\n",
       "     500,\n",
       "     722.2,\n",
       "     722.2,\n",
       "     777.8,\n",
       "     777.8,\n",
       "     611.1,\n",
       "     798.5,\n",
       "     656.8,\n",
       "     526.5,\n",
       "     771.4,\n",
       "     527.8,\n",
       "     718.7,\n",
       "     594.9,\n",
       "     844.5,\n",
       "     544.5,\n",
       "     677.8,\n",
       "     762,\n",
       "     689.7,\n",
       "     1200.9,\n",
       "     820.5,\n",
       "     796.1,\n",
       "     695.6,\n",
       "     816.7,\n",
       "     847.5,\n",
       "     605.6,\n",
       "     544.6,\n",
       "     625.8,\n",
       "     612.8,\n",
       "     987.8,\n",
       "     713.3,\n",
       "     668.3,\n",
       "     724.7,\n",
       "     666.7,\n",
       "     666.7,\n",
       "     666.7,\n",
       "     666.7,\n",
       "     666.7,\n",
       "     611.1,\n",
       "     611.1,\n",
       "     444.4,\n",
       "     444.4,\n",
       "     444.4,\n",
       "     444.4,\n",
       "     500,\n",
       "     500,\n",
       "     388.9,\n",
       "     388.9,\n",
       "     277.8,\n",
       "     500,\n",
       "     500,\n",
       "     611.1,\n",
       "     500,\n",
       "     277.8,\n",
       "     833.3,\n",
       "     750,\n",
       "     833.3]},\n",
       "   '/F31': {'/Type': '/Font',\n",
       "    '/Subtype': '/Type1',\n",
       "    '/BaseFont': '/QJHRZL+CMTT10',\n",
       "    '/FontDescriptor': {'/Type': '/FontDescriptor',\n",
       "     '/FontName': '/QJHRZL+CMTT10',\n",
       "     '/Flags': 4,\n",
       "     '/FontBBox': [-4, -233, 537, 696],\n",
       "     '/Ascent': 611,\n",
       "     '/CapHeight': 611,\n",
       "     '/Descent': -222,\n",
       "     '/ItalicAngle': 0,\n",
       "     '/StemV': 69,\n",
       "     '/XHeight': 431,\n",
       "     '/CharSet': '/A/B/C/D/E/F/G/H/I/J/K/L/M/N/O/P/Q/R/S/T/U/V/W/X/Y/Z/a/ampersand/asciicircum/asciitilde/asterisk/at/b/backslash/bar/braceleft/braceright/bracketleft/bracketright/c/colon/comma/d/e/eight/equal/exclam/f/five/four/g/greater/h/hyphen/i/j/k/l/less/m/n/nine/numbersign/o/one/p/parenleft/parenright/percent/period/plus/q/question/quotedbl/quoteleft/quoteright/r/s/semicolon/seven/six/slash/t/three/two/u/underscore/v/w/x/y/z/zero',\n",
       "     '/FontFile': {'/Length1': 2919,\n",
       "      '/Length2': 20917,\n",
       "      '/Length3': 0,\n",
       "      '/Filter': '/FlateDecode'}},\n",
       "    '/FirstChar': 33,\n",
       "    '/LastChar': 126,\n",
       "    '/Widths': [525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525,\n",
       "     525]},\n",
       "   '/F10': {'/Type': '/Font',\n",
       "    '/Subtype': '/Type1',\n",
       "    '/BaseFont': '/RTSOAF+CMMI7',\n",
       "    '/FontDescriptor': {'/Type': '/FontDescriptor',\n",
       "     '/FontName': '/RTSOAF+CMMI7',\n",
       "     '/Flags': 4,\n",
       "     '/FontBBox': [-1, -250, 1171, 750],\n",
       "     '/Ascent': 694,\n",
       "     '/CapHeight': 683,\n",
       "     '/Descent': -194,\n",
       "     '/ItalicAngle': -14,\n",
       "     '/StemV': 81,\n",
       "     '/XHeight': 431,\n",
       "     '/CharSet': '/F/K/M/N/T/a/comma/i/j/k/m/n/p/period/r/s/slash/t/theta/x/y',\n",
       "     '/FontFile': {'/Length1': 1692,\n",
       "      '/Length2': 10410,\n",
       "      '/Length3': 0,\n",
       "      '/Filter': '/FlateDecode'}},\n",
       "    '/FirstChar': 18,\n",
       "    '/LastChar': 121,\n",
       "    '/Widths': [544.1,\n",
       "     422.8,\n",
       "     668.8,\n",
       "     677.6,\n",
       "     694.6,\n",
       "     572.8,\n",
       "     519.8,\n",
       "     668,\n",
       "     592.7,\n",
       "     662,\n",
       "     526.8,\n",
       "     632.9,\n",
       "     686.9,\n",
       "     713.8,\n",
       "     756,\n",
       "     719.7,\n",
       "     539.7,\n",
       "     689.9,\n",
       "     950,\n",
       "     592.7,\n",
       "     439.2,\n",
       "     751.4,\n",
       "     1138.9,\n",
       "     1138.9,\n",
       "     1138.9,\n",
       "     1138.9,\n",
       "     339.3,\n",
       "     339.3,\n",
       "     585.3,\n",
       "     585.3,\n",
       "     585.3,\n",
       "     585.3,\n",
       "     585.3,\n",
       "     585.3,\n",
       "     585.3,\n",
       "     585.3,\n",
       "     585.3,\n",
       "     585.3,\n",
       "     585.3,\n",
       "     585.3,\n",
       "     339.3,\n",
       "     339.3,\n",
       "     892.9,\n",
       "     585.3,\n",
       "     892.9,\n",
       "     585.3,\n",
       "     610.1,\n",
       "     859.1,\n",
       "     863.2,\n",
       "     819.4,\n",
       "     934.1,\n",
       "     838.7,\n",
       "     724.5,\n",
       "     889.4,\n",
       "     935.6,\n",
       "     506.3,\n",
       "     632,\n",
       "     959.9,\n",
       "     783.7,\n",
       "     1089.4,\n",
       "     904.9,\n",
       "     868.9,\n",
       "     727.3,\n",
       "     899.7,\n",
       "     860.6,\n",
       "     701.5,\n",
       "     674.8,\n",
       "     778.2,\n",
       "     674.6,\n",
       "     1074.4,\n",
       "     936.9,\n",
       "     671.5,\n",
       "     778.4,\n",
       "     462.3,\n",
       "     462.3,\n",
       "     462.3,\n",
       "     1138.9,\n",
       "     1138.9,\n",
       "     478.2,\n",
       "     619.7,\n",
       "     502.4,\n",
       "     510.5,\n",
       "     594.7,\n",
       "     542,\n",
       "     557.1,\n",
       "     557.3,\n",
       "     668.8,\n",
       "     404.2,\n",
       "     472.7,\n",
       "     607.3,\n",
       "     361.3,\n",
       "     1013.7,\n",
       "     706.2,\n",
       "     563.9,\n",
       "     588.9,\n",
       "     523.6,\n",
       "     530.4,\n",
       "     539.2,\n",
       "     431.6,\n",
       "     675.4,\n",
       "     571.4,\n",
       "     826.4,\n",
       "     647.8,\n",
       "     579.4]},\n",
       "   '/F38': {'/Type': '/Font',\n",
       "    '/Subtype': '/Type1',\n",
       "    '/BaseFont': '/WOUVYJ+SFTT1000',\n",
       "    '/FontDescriptor': {'/Type': '/FontDescriptor',\n",
       "     '/FontName': '/WOUVYJ+SFTT1000',\n",
       "     '/Flags': 4,\n",
       "     '/FontBBox': [-208, -360, 1374, 838],\n",
       "     '/Ascent': 0,\n",
       "     '/CapHeight': 0,\n",
       "     '/Descent': 0,\n",
       "     '/ItalicAngle': 0,\n",
       "     '/StemV': 50,\n",
       "     '/XHeight': 430,\n",
       "     '/CharSet': '/multiply/quotesingle',\n",
       "     '/FontFile': {'/Length1': 725,\n",
       "      '/Length2': 16191,\n",
       "      '/Length3': 0,\n",
       "      '/Filter': '/FlateDecode'}},\n",
       "    '/FirstChar': 39,\n",
       "    '/LastChar': 214,\n",
       "    '/Widths': [524.9,\n",
       "     0,\n",
       "     0,\n",
       "     524.9,\n",
       "     0,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     0,\n",
       "     0,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     524.9,\n",
       "     0,\n",
       "     524.9,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     524.9,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     524.9,\n",
       "     0,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     0,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     641.5,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     524.9,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     0,\n",
       "     524.9],\n",
       "    '/Encoding': {'/Type': '/Encoding',\n",
       "     '/Differences': [36,\n",
       "      '/dollar',\n",
       "      39,\n",
       "      '/quotesingle',\n",
       "      136,\n",
       "      '/bullet',\n",
       "      167,\n",
       "      '/section',\n",
       "      214,\n",
       "      '/multiply']}}},\n",
       "  '/ProcSet': ['/PDF', '/Text']},\n",
       " '/MediaBox': [0, 0, 612, 792],\n",
       " '/Parent': {'/Type': '/Pages',\n",
       "  '/Count': 6,\n",
       "  '/Parent': {'/Type': '/Pages',\n",
       "   '/Count': 36,\n",
       "   '/Parent': {'/Type': '/Pages',\n",
       "    '/Count': 178,\n",
       "    '/Kids': [IndirectObject(1993, 0),\n",
       "     IndirectObject(1994, 0),\n",
       "     IndirectObject(1995, 0),\n",
       "     IndirectObject(1996, 0),\n",
       "     IndirectObject(1998, 0)]},\n",
       "   '/Kids': [IndirectObject(432, 0),\n",
       "    IndirectObject(568, 0),\n",
       "    IndirectObject(606, 0),\n",
       "    IndirectObject(657, 0),\n",
       "    IndirectObject(695, 0),\n",
       "    IndirectObject(731, 0)]},\n",
       "  '/Kids': [IndirectObject(689, 0),\n",
       "   IndirectObject(698, 0),\n",
       "   IndirectObject(703, 0),\n",
       "   IndirectObject(707, 0),\n",
       "   IndirectObject(711, 0),\n",
       "   IndirectObject(718, 0)]}}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting slate3k\n",
      "  Downloading https://files.pythonhosted.org/packages/cb/e3/f27cac1dd24617894cf7ddb5da13beca27c9236736466bebaf5dd2a902c1/slate3k-0.5.3-py2.py3-none-any.whl\n",
      "Collecting pdfminer3k (from slate3k)\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/8c/87/cee0aa24f95c287020df7e3936cb51d32b34b05b430759bac15f89ea5ac2/pdfminer3k-1.3.1.tar.gz (4.1MB)\n",
      "\u001b[K     |████████████████████████████████| 4.1MB 816kB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: pytest>=2.0 in /Users/vb/anaconda3/lib/python3.7/site-packages (from pdfminer3k->slate3k) (5.0.1)\n",
      "Requirement already satisfied: ply>=3.4 in /Users/vb/anaconda3/lib/python3.7/site-packages (from pdfminer3k->slate3k) (3.11)\n",
      "Requirement already satisfied: py>=1.5.0 in /Users/vb/anaconda3/lib/python3.7/site-packages (from pytest>=2.0->pdfminer3k->slate3k) (1.8.0)\n",
      "Requirement already satisfied: packaging in /Users/vb/anaconda3/lib/python3.7/site-packages (from pytest>=2.0->pdfminer3k->slate3k) (19.1)\n",
      "Requirement already satisfied: attrs>=17.4.0 in /Users/vb/anaconda3/lib/python3.7/site-packages (from pytest>=2.0->pdfminer3k->slate3k) (19.1.0)\n",
      "Requirement already satisfied: more-itertools>=4.0.0 in /Users/vb/anaconda3/lib/python3.7/site-packages (from pytest>=2.0->pdfminer3k->slate3k) (7.2.0)\n",
      "Requirement already satisfied: atomicwrites>=1.0 in /Users/vb/anaconda3/lib/python3.7/site-packages (from pytest>=2.0->pdfminer3k->slate3k) (1.3.0)\n",
      "Requirement already satisfied: pluggy<1.0,>=0.12 in /Users/vb/anaconda3/lib/python3.7/site-packages (from pytest>=2.0->pdfminer3k->slate3k) (0.12.0)\n",
      "Requirement already satisfied: importlib-metadata>=0.12 in /Users/vb/anaconda3/lib/python3.7/site-packages (from pytest>=2.0->pdfminer3k->slate3k) (0.19)\n",
      "Requirement already satisfied: wcwidth in /Users/vb/anaconda3/lib/python3.7/site-packages (from pytest>=2.0->pdfminer3k->slate3k) (0.1.7)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /Users/vb/anaconda3/lib/python3.7/site-packages (from packaging->pytest>=2.0->pdfminer3k->slate3k) (2.4.2)\n",
      "Requirement already satisfied: six in /Users/vb/anaconda3/lib/python3.7/site-packages (from packaging->pytest>=2.0->pdfminer3k->slate3k) (1.12.0)\n",
      "Requirement already satisfied: zipp>=0.5 in /Users/vb/anaconda3/lib/python3.7/site-packages (from importlib-metadata>=0.12->pytest>=2.0->pdfminer3k->slate3k) (0.5.2)\n",
      "Building wheels for collected packages: pdfminer3k\n",
      "  Building wheel for pdfminer3k (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for pdfminer3k: filename=pdfminer3k-1.3.1-cp37-none-any.whl size=105358 sha256=e89eaaa7c6d0dcfcde0e51ceeb81a33c91583e11c320b7879b565f405ee3ce6d\n",
      "  Stored in directory: /Users/vb/Library/Caches/pip/wheels/ca/4f/a7/cb601b4fb257d2321ac668b7c6e269176780bd0283eda855d2\n",
      "Successfully built pdfminer3k\n",
      "Installing collected packages: pdfminer3k, slate3k\n",
      "Successfully installed pdfminer3k-1.3.1 slate3k-0.5.3\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install slate3k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:pdfminer.layout:Too many boxes (740) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (1024) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (719) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (101) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (115) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (106) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (113) to group, skipping.\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.layout:Too many boxes (111) to group, skipping.\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.layout:Too many boxes (109) to group, skipping.\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.layout:Too many boxes (115) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (158) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (123) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (107) to group, skipping.\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 20\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 21\n",
      "WARNING:pdfminer.layout:Too many boxes (101) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (116) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (154) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (110) to group, skipping.\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 88\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 88\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.layout:Too many boxes (107) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (138) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (143) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (125) to group, skipping.\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 34\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 35\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 34\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 112\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 112\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 35\n",
      "WARNING:pdfminer.layout:Too many boxes (138) to group, skipping.\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.layout:Too many boxes (106) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (130) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (148) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (111) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (104) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (120) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (101) to group, skipping.\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.layout:Too many boxes (156) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (154) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (156) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (109) to group, skipping.\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 26\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 54\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='LHNROQ+CMSY9'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='LHNROQ+CMSY9'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='LHNROQ+CMSY9'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='LHNROQ+CMSY9'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 88\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 88\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 88\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 88\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 20\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 21\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='JXANHS+CMSY7'>, 48\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='JXANHS+CMSY7'>, 48\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='NQLEEW+CMSY6'>, 48\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='NQLEEW+CMSY6'>, 48\n",
      "WARNING:pdfminer.layout:Too many boxes (102) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (146) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (129) to group, skipping.\n",
      "WARNING:pdfminer.layout:Too many boxes (156) to group, skipping.\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 20\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 21\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 20\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 21\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='LHNROQ+CMSY9'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='LHNROQ+CMSY9'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.layout:Too many boxes (168) to group, skipping.\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='LHNROQ+CMSY9'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='LHNROQ+CMSY9'>, 107\n",
      "WARNING:pdfminer.layout:Too many boxes (155) to group, skipping.\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 20\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 21\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 20\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 21\n",
      "WARNING:pdfminer.layout:Too many boxes (117) to group, skipping.\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 20\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 112\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='RASHKQ+CMEX10'>, 21\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.converter:undefined: <PDFType1Font: basefont='OIMRRQ+CMSY10'>, 107\n",
      "WARNING:pdfminer.layout:Too many boxes (121) to group, skipping.\n"
     ]
    }
   ],
   "source": [
    "import slate3k as slate\n",
    "\n",
    "with open(\"companion.pdf\",'rb') as f:\n",
    "    extracted_text = slate.PDF(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Introduction to Applied Linear Algebra\\n\\nVectors, Matrices, and Least Squares\\n\\nJulia Language Companion\\n\\nStephen Boyd and Lieven Vandenberghe\\n\\nDRAFT August 26, 2018\\n\\n\\x0c',\n",
       " '\\x0c',\n",
       " 'Contents\\n\\nPreface\\n\\nvii\\n\\nGetting started with Julia\\n\\nix\\n\\n1\\n\\nVectors\\n\\n1\\n\\n1.1\\n\\nVectors .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n1\\n\\n1.2\\n\\nVector addition\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n9\\n\\n1.3\\n\\nScalar-vector multiplication\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n9\\n\\n1.4\\n\\nInner product\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n14\\n\\n1.5\\n\\nComplexity of vector computations\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n15\\n\\n2\\n\\nLinear functions\\n\\n19\\n\\n2.1\\n\\nLinear functions\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n19\\n\\n2.2\\n\\nTaylor approximation\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n20\\n\\n2.3\\n\\nRegression model\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n21\\n\\n3\\n\\nNorm and distance\\n\\n25\\n\\n3.1\\n\\nNorm .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n25\\n\\n3.2\\n\\nDistance\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n27\\n\\n3.3\\n\\nStandard deviation .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n28\\n\\n3.4\\n\\nAngle .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n29\\n\\n3.5\\n\\nComplexity .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n30\\n\\n4\\n\\nClustering\\n\\n33\\n\\n4.1\\n\\nClustering\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n33\\n\\n4.2\\n\\nA clustering objective\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n33\\n\\n4.3\\n\\nThe k-means algorithm .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n34\\n\\n4.4\\n\\nExamples .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n37\\n\\n4.5\\n\\nApplications\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n37\\n\\n5\\n\\nLinear independence\\n\\n39\\n\\n5.1\\n\\nLinear dependence .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n39\\n\\n5.2\\n\\nBasis\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n39\\n\\n5.3\\n\\nOrthonormal vectors .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n40\\n\\n5.4\\n\\nGram–Schmidt algorithm .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n41\\n\\n\\x0c',\n",
       " 'iv\\n\\nContents\\n\\n6 Matrices\\n\\n45\\n\\n6.1\\n\\nMatrices\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n45\\n\\n6.2\\n\\nZero and identity matrices\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n50\\n\\n6.3\\n\\nTranspose, addition, and norm\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n54\\n\\n6.4\\n\\nMatrix-vector multiplication .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n56\\n\\n6.5\\n\\nComplexity .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n59\\n\\n7 Matrix examples\\n\\n61\\n\\n7.1\\n\\nGeometric transformations\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n61\\n\\n7.2\\n\\nSelectors\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n61\\n\\n7.3\\n\\nIncidence matrix\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n63\\n\\n7.4\\n\\nConvolution\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n65\\n\\n8\\n\\nLinear equations\\n\\n67\\n\\n8.1\\n\\nLinear and aﬃne functions\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n67\\n\\n8.2\\n\\nLinear function models\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n69\\n\\n8.3\\n\\nSystems of\\n\\nlinear equations\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n71\\n\\n9\\n\\nLinear dynamical systems\\n\\n73\\n\\n9.1\\n\\nLinear dynamical systems\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n73\\n\\n9.2\\n\\nPopulation dynamics\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n74\\n\\n9.3\\n\\nEpidemic dynamics\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n75\\n\\n9.4\\n\\nMotion of a mass\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n75\\n\\n9.5\\n\\nSupply chain dynamics\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n76\\n\\n10 Matrix multiplication\\n\\n79\\n\\n10.1 Matrix-matrix multiplication\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n79\\n\\n10.2 Composition of\\n\\nlinear functions\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n80\\n\\n10.3 Matrix power\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n81\\n\\n10.4 QR factorization .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n82\\n\\n11 Matrix inverses\\n\\n85\\n\\n11.1 Left and right inverses .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n85\\n\\n11.2\\n\\nInverse\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n86\\n\\n11.3 Solving linear equations\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n87\\n\\n11.4 Examples .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n90\\n\\n11.5 Pseudo-inverse .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n91\\n\\n12 Least squares\\n\\n93\\n\\n12.1 Least squares problem .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n93\\n\\n12.2 Solution\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n94\\n\\n12.3 Solving least squares problems\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n95\\n\\n12.4 Examples .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n96\\n\\n13 Least squares data ﬁtting\\n\\n99\\n\\n13.1 Least squares data ﬁtting\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n99\\n\\n13.2 Validation\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 105\\n\\n13.3 Feature engineering\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 108\\n\\n\\x0c',\n",
       " 'Contents\\n\\nv\\n\\n14 Least squares classiﬁcation\\n\\n113\\n\\n14.1 Classiﬁcation .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 113\\n\\n14.2 Least squares classiﬁer .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 114\\n\\n14.3 Multi-class classiﬁers\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 115\\n\\n15 Multi-objective least squares\\n\\n121\\n\\n15.1 Multi-objective least squares\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 121\\n\\n15.2 Control\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 124\\n\\n15.3 Estimation and inversion\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 124\\n\\n15.4 Regularized data ﬁtting\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 125\\n\\n15.5 Complexity .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 127\\n\\n16 Constrained least squares\\n\\n129\\n\\n16.1 Constrained least squares problem\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 129\\n\\n16.2 Solution\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 131\\n\\n16.3 Solving contrained least squares problems\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 132\\n\\n17 Constrained least squares applications\\n\\n135\\n\\n17.1 Portfolio optimization\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 135\\n\\n17.2 Linear quadratic control\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 137\\n\\n17.3 Linear quadratic state estimation\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 143\\n\\n18 Nonlinear least squares\\n\\n145\\n\\n18.1 Nonlinear equations and least squares\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 145\\n\\n18.2 Gauss–Newton algorithm\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 145\\n\\n18.3 Levenberg–Marquardt algorithm\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 147\\n\\n18.4 Nonlinear model ﬁtting\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 152\\n\\n18.5 Nonlinear least squares classiﬁcation\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 155\\n\\n19 Constrained nonlinear least squares\\n\\n157\\n\\n19.1 Constrained nonlinear least squares\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 157\\n\\n19.2 Penalty algorithm .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 157\\n\\n19.3 Augmented Lagrangian algorithm .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 159\\n\\n19.4 Nonlinear control\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 161\\n\\nAppendices\\n\\n163\\n\\nA\\n\\nThe VMLS package\\n\\n165\\n\\nA.1\\n\\nUtility functions\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 165\\n\\nA.2\\n\\nAlgorithms\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 166\\n\\nA.3\\n\\nData sets .\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n.\\n\\n. 167\\n\\n\\x0c',\n",
       " 'vi\\n\\nContents\\n\\n\\x0c',\n",
       " 'Preface\\n\\nThis Julia Language Companion accompanies our book Introduction to Applied\\n\\nLinear Algebra: Vectors, Matrices, and Least Squares (referred to here as VMLS).\\n\\nIt\\n\\nis meant\\n\\nto show how the ideas and methods\\n\\nin VMLS can be expressed and\\n\\nimplemented in the programming language Julia.\\n\\nWe assume that the reader has installed Julia, or is using Juliabox online, and\\n\\nunderstands the basics of the language. We also assume that the reader has care-\\n\\nfully read the relevant sections of VMLS. The organization of the Julia Language\\n\\nCompanion follows VMLS, section by section.\\n\\nYou will see that mathematical notation and Julia syntax are pretty close, but\\n\\nnot\\n\\nthe same. You must be careful\\n\\nto never confuse mathematical notation and\\n\\nJulia syntax.\\n\\nIn these notes we write mathematical notation (as\\n\\nin VMLS)\\n\\nin\\n\\nstandard mathematics\\n\\nfont, e.g., y = Ax.\\n\\nJulia code, expressions, and snippets\\n\\nare written in a ﬁxed-width typewriter font, e.g., y\\n\\n=\\n\\nA*x. We encourage you to\\n\\ncut and paste our Julia code snippets into a Julia interactive session or notebook,\\n\\nto test them out, and maybe modify them and run them again. You can also use\\n\\nour snippets as templates for your own Julia code. At some point we will collect\\n\\nthe Julia snippets in this companion document into Julia notebooks that you can\\n\\neasily run.\\n\\nJulia is a very powerful\\n\\nlanguage, but in this companion document we use only\\n\\na small and limited set of\\n\\nits\\n\\nfeatures. The code snippets\\n\\nin this companion are\\n\\nwritten so as to to be transparent and simple, and to emphasize closeness to the\\n\\nconcepts\\n\\nin VMLS. Some of\\n\\nthe code snippets could be written in a much more\\n\\ncompact way, or in a way that would result in faster or more eﬃcient execution.\\n\\nThe code snippets\\n\\nin this compaion document are compatible with Julia 1.0,\\n\\nwhich is not quite the same as earlier versions,\\n\\nlike Julia 0.6. Some of the functions\\n\\nwe use are in the standard packages LinearAlgebra, SparseArrays, and Plots,\\n\\nand a few others. We have created a simple and small Julia package VMLS, which\\n\\nincludes a small number of simple Julia functions that are useful\\n\\nfor studying the\\n\\nmaterial\\n\\nin VMLS. The next section, Getting started with Julia, explains how to\\n\\ninstall these Julia packages.\\n\\nWe consider\\n\\nthis companion to be a draft. We’ll be updating it occasionally,\\n\\nadding more examples and ﬁxing typos as we become aware of them. So you may\\n\\nwish to periodically check whether a newer version is available.\\n\\nStephen Boyd\\n\\nLieven Vandenberghe\\n\\nStanford, California\\n\\nLos Angeles, California\\n\\n\\x0c',\n",
       " '\\x0c',\n",
       " 'Getting started with Julia\\n\\nInstalling Julia.\\n\\nDownload Julia 1.0 from its website, and then follow the instruc-\\n\\ntions to install\\n\\nit on your platform. You’ll want to make sure it’s working before\\n\\nproceeding to install additional packages, as described below.\\n\\nInstalling packages. While most of the Julia code we use in this companion is from\\n\\nthe base or core of the Julia language,\\n\\nseveral\\n\\nimportant functions are contained\\n\\nin other packages that you must explicitly install. Here we explain how to do this.\\n\\nThe installation has to be done only once.\\n\\nTo add (install) a Julia package you use the package manager\\n\\nsystem, which\\n\\nis\\n\\nentered by typing ] to the Julia prompt.\\n\\nYou’ll\\n\\nthen see\\n\\nthe package man-\\n\\nager prompt, (v.10) pkg>.\\n\\nIf you type ?, you’ll\\n\\nsee a list of package manager\\n\\ncommands. You exit\\n\\nthe package manager by typing Ctrl-C (Control-C), which\\n\\nreturns you to the julia prompt.\\n\\nTo add a package called PackageName, type add\\n\\nPackageName to the package\\n\\ncontrol prompt.\\n\\nIt may take a few minutes to get the required packages, compile,\\n\\nand install them.\\n\\nIf you type status, you’ll see what packages are installed.\\n\\nIf you\\n\\ntype up,\\n\\ninstalled packages will be upgraded,\\n\\nif needed.\\n\\nThis companion will use functions from the following packages:\\n\\n• LinearAlgebra\\n\\n• SparseArrays\\n\\n• Plots\\n\\nOn a few occasions we use functions\\n\\nfrom the packages Random and DSP, but we\\n\\nwill mention this in the text when we use these functions.\\n\\nHere’s what it looks like when you install the Plots package.\\n\\njulia>\\n\\n]\\n\\n(v1.0)\\n\\npkg>\\n\\nadd\\n\\nPlots\\n\\nUpdating registry\\n\\nat\\n\\n‘~/.julia/registries/General‘\\n\\nUpdating git-repo\\n\\n‘https://github.com/JuliaRegistries/General.git‘\\n\\nResolving\\n\\npackage\\n\\nversions...\\n\\nUpdating ‘~/.julia/environments/v1.0/Project.toml‘\\n\\n[no\\n\\nchanges]\\n\\n\\x0c',\n",
       " 'x\\n\\nGetting started with Julia\\n\\nUpdating ‘~/.julia/environments/v1.0/Manifest.toml‘\\n\\n[no changes]\\n\\n(v1.0) pkg> status\\n\\nStatus\\n\\n‘~/.julia/environments/v1.0/Project.toml‘\\n\\n[91a5bcdd] Plots\\n\\nv0.19.3\\n\\n[011e45e9] SparseArrays\\n\\nv0.4.2\\n\\n[37e2e46d] LinearAlgebra\\n\\n(v1.0)\\n\\npkg> ^C\\n\\njulia>\\n\\nThe VMLS package. We have created a small package called VMLS.\\n\\nIt contains a\\n\\nfew functions that use notation closer to VMLS notation or are easier to use than\\n\\nthe\\n\\ncorresponding Julia functions, basic\\n\\nimplementations of\\n\\nsome algorithms\\n\\nin\\n\\nVMLS, and functions that generate data used in examples. The list of\\n\\nfunctions is\\n\\ngiven in Appendix A. To install VMLS, go to the package manager from the Julia\\n\\nprompt, then install\\n\\nit as follows.\\n\\njulia> ]\\n\\npkg>\\n\\nadd\\n\\nhttps://github.com/VMLS-book/VMLS.jl\\n\\npkg>\\n\\n^C\\n\\njulia>\\n\\nUsing packages.\\n\\nOnce a package is installed (which needs to be done only once),\\n\\nyou import it into Julia with the command using,\\n\\nfollowed by the package name,\\n\\nor a comma separated list of package names. This too can take some time. After\\n\\nexecuting this command you can access the functions contained in the packages.\\n\\nTo run any the code fragments in this companion you will need to ﬁrst execute\\n\\nthe statement\\n\\njulia> using LinearAlgebra,\\n\\nSparseArrays,\\n\\nVMLS\\n\\nWhen we use other packages (in particular, Plots), we include the using statement\\n\\nin the code.\\n\\n\\x0c',\n",
       " 'Chapter 1\\n\\nVectors\\n\\n1.1\\n\\nVectors\\n\\nVectors\\n\\nin Julia are represented by one-dimensional Array ob jects.\\n\\nA vector\\n\\nis\\n\\nconstructed by giving the list of elements surrounded by square brackets, with the\\n\\nelements separated by commas or semicolons. The assignment operator = is used\\n\\nto give a name to the array. The length function returns the size (dimension).\\n\\njulia>\\n\\nx\\n\\n= [ -1.1,\\n\\n0.0,\\n\\n3.6,\\n\\n-7.2\\n\\n]\\n\\n4-element Array{Float64,1}:\\n\\n-1.1\\n\\n0.0\\n\\n3.6\\n\\n-7.2\\n\\njulia>\\n\\nlength(x)\\n\\n4\\n\\njulia>\\n\\ny\\n\\n= [ -1.1;\\n\\n0.0;\\n\\n3.6;\\n\\n-7.2\\n\\n]\\n\\n#\\n\\nUsing\\n\\nsemicolons\\n\\n4-element Array{Float64,1}:\\n\\n-1.1\\n\\n0.0\\n\\n3.6\\n\\n-7.2\\n\\njulia>\\n\\nlength(y)\\n\\n4\\n\\nThe Array{Float64,1} displayed by Julia above each array tells us that the array\\n\\nis one-dimensional and its entries are ﬂoating point numbers that use 64 bits.\\n\\nSome common mistakes.\\n\\nDon’t forget the commas or semicolons between entries,\\n\\nand be sure to use square brackets and not parentheses. Otherwise you’ll get things\\n\\nthat makes sense in Julia, but are not vectors.\\n\\n\\x0c',\n",
       " '2\\n\\n1\\n\\nVectors\\n\\njulia> a\\n1×2\\n\\nArray{Int64,2}:\\n\\n=\\n\\n[ 1\\n\\n2\\n\\n]\\n\\n1\\n\\n2\\n\\njulia> b\\n\\n=\\n\\n( 1,\\n\\n2\\n\\n)\\n\\n(1, 2)\\n\\nHere a is a row vector, which we will encounter later; b is a tuple or list consisting\\n\\nof two scalars.\\n\\nIndexing.\\n\\nA speciﬁc element xi\\nindex (which runs to 1 to n,\\nfor an n-vector). Array indexing can be also be used\\n\\nis retrieved by the expression x[i] where i is the\\n\\non the left-hand side of an assignment, to change the value of a speciﬁc element.\\n\\njulia> x\\n\\n=\\n\\n[ -1.1,\\n\\n0.0,\\n\\n3.6,\\n\\n-7.2\\n\\n];\\n\\njulia> x[3]\\n\\n3.6\\n\\njulia> x[3] =\\n\\n4.0;\\n\\njulia> x\\n\\n4-element\\n\\nArray{Float64,1}:\\n\\n-1.1\\n\\n0.0\\n\\n4.0\\n\\n-7.2\\n\\nThe special\\n\\nindex end refers to the last index of a vector.\\n\\nIn the example above,\\n\\nx[end] and x[length(x)] both give the last entry, -7.2.\\n\\nAssignment versus\\n\\ncopying.\\n\\nMatlab or Octave users may be surprised by the\\n\\nbehavior of an assignment y\\n\\n=\\n\\nx if x is an array. This expression gives a new name\\n\\n(or\\n\\nreference) y to the same array already referenced by x.\\n\\nIt does not create a\\n\\nnew copy of the array x.\\n\\njulia> x\\n\\njulia> y\\n\\n=\\n\\n=\\n\\n[ -1.1,\\n\\n0.0,\\n\\n3.6,\\n\\n-7.2\\n\\n];\\n\\nx;\\n\\njulia> x[3] =\\n\\n4.0;\\n\\njulia> y\\n\\n4-element\\n\\nArray{Float64,1}:\\n\\n-1.1\\n\\n0.0\\n\\n4.0\\n\\n-7.2\\n\\n# The\\n\\nassignment\\n\\nto\\n\\nx[3]\\n\\nalso\\n\\nchanged\\n\\ny[3]\\n\\njulia> y[1] =\\n\\n2.0;\\n\\njulia> x\\n\\n\\x0c',\n",
       " '1.1\\n\\nVectors\\n\\n3\\n\\n4-element Array{Float64,1}:\\n\\n# The\\n\\nassignment\\n\\nto\\n\\ny[1]\\n\\nalso\\n\\nchanged\\n\\nx[1]\\n\\n2.0\\n\\n0.0\\n\\n4.0\\n\\n-7.2\\n\\nTo create a new copy of an array, the function copy should be used.\\n\\njulia>\\n\\njulia>\\n\\nx\\n\\ny\\n\\n= [ -1.1,\\n\\n0.0,\\n\\n3.6,\\n\\n-7.2\\n\\n];\\n\\n= copy(x);\\n\\njulia>\\n\\nx[3]\\n\\n= 4.0;\\n\\njulia>\\n\\ny\\n\\n4-element Array{Float64,1}:\\n\\n-1.1\\n\\n0.0\\n\\n3.6\\n\\n-7.2\\n\\njulia>\\n\\ny[1]\\n\\n= 2.0;\\n\\njulia>\\n\\nx\\n\\n4-element Array{Float64,1}:\\n\\n-1.1\\n\\n0.0\\n\\n4.0\\n\\n-7.2\\n\\nVector equality.\\n\\nEquality of vectors is checked using the relational operator ==.\\n\\nFor\\n\\ntwo vectors\\n\\n(arrays) a and b,\\n\\nthe Julia expression a==b evaluates\\n\\nto true if\\n\\nthe vectors (arrays) are equal, i.e., they have the same length and identical entries,\\n\\nand false otherwise.\\n\\njulia>\\n\\njulia>\\n\\nx\\n\\ny\\n\\n= [ -1.1,\\n\\n0.0,\\n\\n3.6,\\n\\n-7.2\\n\\n];\\n\\n= copy(x);\\n\\njulia>\\n\\ny[3]\\n\\n= 4.0\\n\\njulia>\\n\\ny\\n\\n==\\n\\nx\\n\\nfalse\\n\\njulia>\\n\\nz\\n\\n= x\\n\\njulia>\\n\\nz[3]\\n\\n= 4.0\\n\\njulia>\\n\\nz\\n\\n==\\n\\nx\\n\\ntrue\\n\\nScalars versus 1-vectors.\\n\\nIn the mathematical notation used in VMLS we con-\\n\\nsider a 1-vector to be the same as a number. But in Julia, 1-vectors are not the same\\n\\n\\x0c',\n",
       " '4\\n\\n1\\n\\nVectors\\n\\nas scalars (numbers). Julia distinguishes between the 1-vector (array) [\\n\\n1.3 ] and\\n\\nthe number 1.3.\\n\\njulia> x\\n\\n=\\n\\n[ 1.3\\n\\n]\\n\\n1-element\\n\\nArray{Float64,1}:\\n\\n1.3\\n\\njulia> y\\n\\n=\\n\\n1.3\\n\\n1.3\\n\\njulia> x\\n\\n== y\\n\\nfalse\\n\\njulia> x[1] ==\\n\\ny\\n\\ntrue\\n\\nIn the last line, x[1] is the ﬁrst (and only) entry of x, which is indeed the number\\n\\n1.3.\\n\\nBlock or stacked vectors.\\n\\nTo construct a block vector in Julia, you can use vcat\\n\\n(vertical\\nconcatenate) or\\nthe\\nsemicolon (;)\\noperator.\\nLet’s\\nconstruct\\nthe block\\nvector z = (x, y) with x = (1, −2) and y = (1, 1, 0) using the two methods.\\n\\njulia> x\\n\\njulia> z\\n\\n=\\n\\n=\\n\\n[ 1,\\n\\n-2\\n\\n];\\n\\ny\\n\\n[ x;\\n\\ny\\n\\n]\\n\\n=\\n\\n#\\n\\n5-element\\n\\nArray{Int64,1}:\\n\\n[\\n\\n1,\\n\\n1,\\n\\n0\\n\\n];\\n\\nConcatenate\\n\\nusing\\n\\nsemicolon\\n\\n1\\n\\n-2\\n\\n1\\n\\n1\\n\\n0\\n\\njulia> z\\n\\n=\\n\\nvcat(x,\\n\\ny)\\n\\n#\\n\\nConcatenate\\n\\nusing\\n\\nvcat\\n\\n5-element\\n\\nArray{Int64,1}:\\n\\n1\\n\\n-2\\n\\n1\\n\\n1\\n\\n0\\n\\nAs\\n\\nin mathematical notation, you can stack vectors with scalars,\\n\\ne.g., [1;x;0]\\n\\ncreates (1, x, 0).\\n\\nSome common mistakes.\\n\\nThere are a few Julia operations that look similar but\\n\\ndo not construct a block or stacked vector. For example, z\\n\\n=\\n\\n(x,y) creates a list\\n\\nor tuple of the two vectors; z\\n\\n=\\n\\n[x,y] creates an array of the two vectors. Both of\\n\\nthese are valid Julia expression, but neither of them is the stacked vector [x;y].\\n\\nSubvectors and slicing.\\n\\nAs in the mathematical notation used in VMLS, the Julia\\n\\nexpression r:s denotes the index range r, r + 1, . . . , s.\\n\\n(It is assumed here that r\\n\\n\\x0c',\n",
       " '1.1\\n\\nVectors\\n\\n5\\n\\nand s are positive integers with r the smaller of the two.)\\n\\nIn VMLS, we use xr:s\\nIn Julia you can extract a\\n\\nto denote the slice of\\n\\nthe vector x from index r to s.\\n\\nsubvector or slice of a vector using an index range as the argument. You can also\\n\\nuse index ranges to assign a slice of a vector.\\n\\njulia>\\n\\nx\\n\\n= [ 9,\\n\\n4,\\n\\n3,\\n\\n0,\\n\\n5\\n\\n];\\n\\njulia>\\n\\ny\\n\\n= x[2:4]\\n\\n3-element Array{Int64,1}:\\n\\n4\\n\\n3\\n\\n0\\n\\njulia>\\n\\nx[4:5] =\\n\\n[\\n\\n-2,\\n\\n-3\\n\\n];\\n\\n#\\n\\nRe-assign\\n\\nthe\\n\\n4\\n\\nand\\n\\n5\\n\\nentries\\n\\nof\\n\\nx\\n\\njulia>\\n\\nx\\n\\n5-element Array{Int64,1}:\\n\\n9\\n\\n4\\n\\n3\\n\\n-2\\n\\n-3\\n\\nJulia indexing into arrays.\\n\\nJulia slicing and subvectoring is much more general\\n\\nthan the mathematical notation we use\\n\\nin VMLS. For\\n\\nexample,\\n\\none\\n\\ncan use a\\n\\nnumber range with a third argument, that gives the stride, which is the increment\\n\\nbetween successive\\n\\nindexes.\\n\\nFor\\n\\nexample,\\n\\nthe\\n\\nindex range 1:2:5 is\\n\\nthe\\n\\nlist of\\n\\nnumbers 1,3,5. The expression x[1:2:5] extracts the 3-vector [9,3,5],\\n\\ni.e.,\\n\\nthe\\n\\nﬁrst, third, and ﬁfth entries of x deﬁned above. You can also use an index range\\n\\nthat\\n\\nruns backward.\\n\\nFor any vector z,\\n\\nthe Julia expression z[end:-1:1] is\\n\\nthe\\n\\nreversed vector,\\n\\ni.e., the vector with the same coeﬃcients, but in opposite order.\\n\\nVector of ﬁrst diﬀerences.\\n\\nLet’s use slicing to create the (n − 1)-vector d deﬁned\\ni = 1, . . . , n − 1, where x is an n-vector. The vector d is\\n\\nby di = xi+1 − xi ,\\ncalled the vector of (ﬁrst) diﬀerences of x.\\n\\nfor\\n\\njulia>\\n\\nx\\n\\n= [ 1,\\n\\n0,\\n\\n0,\\n\\n-2,\\n\\n2\\n\\n];\\n\\njulia>\\n\\nd\\n\\n= x[2:end]\\n\\n-\\n\\nx[1:end-1]\\n\\n4-element Array{Int64,1}:\\n\\n-1\\n\\n0\\n\\n-2\\n\\n4\\n\\nLists of\\n\\nvectors.\\n\\nAn ordered list of n-vectors might be denoted in VMLS as\\na1 , . . . , ak or a(1) , . . . , a(k) , or\\n\\njust as a, b, c. There are several ways\\n\\nto represent\\n\\n\\x0c',\n",
       " '6\\n\\n1\\n\\nVectors\\n\\nlists of vectors in Julia.\\n\\nIf we give the elements of the list, separated by commas,\\n\\nand surrounded by square brackets, we form a one-dimensional array of vectors.\\n\\nIf\\n\\ninstead we use parentheses as delimiters, we obtain a tuple or list.\\n\\njulia> x\\n\\n=\\n\\n[ 1.0,\\n\\n0\\n\\n];\\n\\ny\\n\\n=\\n\\n[\\n\\n1.0,\\n\\n-1.0\\n\\n];\\n\\nz\\n\\n=\\n\\n[\\n\\n0,\\n\\n1.0];\\n\\njulia> list =\\n\\n[\\n\\nx,\\n\\ny,\\n\\nz\\n\\n]\\n\\n3-element\\n\\nArray{Array{Float64,1},1}:\\n\\n[1.0,\\n\\n0.0]\\n\\n[1.0,\\n\\n-1.0]\\n\\n[0.0,\\n\\n1.0]\\n\\njulia> list[2]\\n\\n#\\n\\nSecond\\n\\nelement\\n\\nof\\n\\nlist\\n\\n2-element\\n\\nArray{Float64,1}:\\n\\n1.0\\n\\n-1.0\\n\\njulia>\\n\\nlist =\\n\\n(\\n\\nx,\\n\\ny,\\n\\nz\\n\\n)\\n\\n([1.0,\\n\\n0.0],\\n\\n[1.0,\\n\\n-1.0],\\n\\n[0.0,\\n\\n1.0])\\n\\njulia>\\n\\nlist[3]\\n\\n#\\n\\nThird\\n\\nelement\\n\\nof\\n\\nlist\\n\\n2-element\\n\\nArray{Float64,1}:\\n\\n0.0\\n\\n1.0\\n\\nNote the diﬀerence between [x,\\n\\ny,\\n\\nz] (an array of arrays) and [x;\\n\\ny;\\n\\nz] (an\\n\\narray of numbers, obtained by concatenation). To extract the ith vector from the\\n\\nlist of vectors, use list[i]. To get the j th element or coeﬃcient of the ith vector\\n\\nin the list, use list[i][j].\\n\\nZero vectors.\\n\\nIn Julia a zero vector of dimension n is created using zeros(n).\\n\\njulia> zeros(3)\\n\\n3-element\\n\\nArray{Float64,1}:\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\nThe expression zeros(length(a)) creates a zero vector with the same size as the\\n\\nvector a.\\n\\nUnit vectors.\\n\\nThere is no built-in Julia function for creating ei , the ith unit vector\\nlength n. The following code creates ei , with i = 2 and n = 4.\\n\\nof\\n\\njulia> i\\n\\n=\\n\\n2;\\n\\nn\\n\\n=\\n\\n4;\\n\\njulia> ei\\n\\n= zeros(n);\\n\\n#\\n\\nCreate\\n\\na\\n\\nzero\\n\\nvector\\n\\njulia> ei[i] =\\n\\n1;\\n\\n#\\n\\nSet\\n\\nith\\n\\nentry\\n\\nto\\n\\n1\\n\\njulia> ei\\n\\n\\x0c',\n",
       " '1.1\\n\\nVectors\\n\\n7\\n\\n4-element Array{Float64,1}:\\n\\n0.0\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\nHere’s another way to create ei using concatenation, using a Julia inline func-\\ntion.\\n\\njulia>\\n\\nunit_vector(i,n)\\n\\n=\\n\\n[zeros(i-1);\\n\\n1\\n\\n;\\n\\nzeros(n-i)]\\n\\nunit_vector\\n\\n(generic\\n\\nfunction\\n\\nwith\\n\\n1\\n\\nmethod)\\n\\njulia>\\n\\nunit_vector(2,4)\\n\\n4-element Array{Float64,1}:\\n\\n0.0\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\nOnes vector.\\n\\nIn Julia the ones vector of dimension n, denoted 1n or\\nis created using ones(n).\\n\\nVMLS,\\n\\njust 1 in\\n\\njulia>\\n\\nones(2)\\n\\n2-element Array{Float64,1}:\\n\\n1.0\\n\\n1.0\\n\\nRandom vectors. We do not use or\\n\\nrefer\\n\\nto random vectors\\n\\nin VMLS, which\\n\\ndoes not assume a background in probability.\\n\\nHowever,\\n\\nit\\n\\nis\\n\\nsometimes useful\\n\\nto generate random vectors,\\n\\nfor example to test an identity or some algorithm.\\n\\nIn\\n\\nJulia, rand(n) generates a random vector of length n with entries that are between\\n\\n0 and 1. Each time this function is called or evaluated,\\n\\nit gives a diﬀerent vector.\\n\\nThe variant randn(n) (with the extra ‘n’ for normal) gives an n-vector with entries\\n\\nthat come from a normal (Gaussian) distribution. They can be positive or negative,\\n\\nwith typical values on the order of one. Remember that every time you evaluate\\n\\nthese functions, you get a diﬀerent random vector.\\n\\nIn particular, you will obtain\\n\\ndiﬀerent entries in the vectors below when you run the code.\\n\\njulia>\\n\\nrand(2)\\n\\n2-element Array{Float64,1}:\\n\\n0.831491\\n\\n0.0497708\\n\\njulia>\\n\\nrand(2)\\n\\n\\x0c',\n",
       " '8\\n\\n1\\n\\nVectors\\n\\nFigure 1.1 Hourly temperature in downtown Los Angeles on August 5 and\\n\\n6, 2015 (starting at 12:47AM, ending at 11:47PM).\\n\\n2-element\\n\\nArray{Float64,1}:\\n\\n0.189284\\n\\n0.713467\\n\\njulia> randn(2)\\n\\n2-element\\n\\nArray{Float64,1}:\\n\\n2.44544\\n\\n-0.12134\\n\\nPlotting.\\n\\nThere are several external packages for creating plots in Julia. One such\\n\\npackage is Plots.jl, which you must add (install) via Julia’s package manager\\n\\ncontrol system; see page ix. Assuming the Plots package had been installed, you\\n\\nimport it into Julia for use, using the command using Plots.\\n\\n(This can take some\\n\\ntime.) After\\n\\nthat you can access\\n\\nthe Julia commands\\n\\nthat create or manipulate\\n\\nplots.\\n\\nFor example, we can plot the temperature time series in Figure 1.3 of VMLS\\n\\nusing the code below; the last line saves the plot in the ﬁle temperature.pdf. The\\n\\nresult is shown in Figure 1.1.\\n\\njulia> using Plots\\n\\n#\\n\\nOnly\\n\\nneed\\n\\nto\\n\\ndo\\n\\nthis\\n\\nonce\\n\\nper\\n\\nsession\\n\\n[\\n\\n[\\n\\nInfo: Precompiling\\n\\nPlots\\n\\n[91a5bcdd-55d7-5caf-9e0b-520d859cae80]\\n\\nInfo: Precompiling\\n\\nGR\\n\\n[28b8d3ca-fb5f-59d9-8090-bfdbd6d07a71]\\n\\njulia> temps =\\n\\n[\\n\\n71,\\n\\n71,\\n\\n68,\\n\\n69,\\n\\n68,\\n\\n69,\\n\\n68,\\n\\n74, 77,\\n\\n82,\\n\\n85,\\n\\n86,\\n\\n88, 86, 85, 86,\\n\\n84,\\n\\n79,\\n\\n77,\\n\\n75,\\n\\n73,\\n\\n71,\\n\\n70,\\n\\n70,\\n\\n69,\\n\\n69,\\n\\n69,\\n\\n69, 67, 68, 68,\\n\\n73,\\n\\n76,\\n\\n77,\\n\\n82,\\n\\n84,\\n\\n84,\\n\\n81,\\n\\n80,\\n\\n78,\\n\\n79,\\n\\n78,\\n\\n73, 72, 70, 70,\\n\\n68,\\n\\n67\\n\\n];\\n\\n01020304070758085\\x0c',\n",
       " '1.2\\n\\nVector addition\\n\\n9\\n\\njulia>\\n\\nplot(temps,\\n\\nmarker\\n\\n=\\n\\n:circle,\\n\\nlegend\\n\\n=\\n\\nfalse,\\n\\ngrid\\n\\n=\\n\\nfalse)\\n\\njulia>\\n\\nsavefig(\"temperature.pdf\")\\n\\n1.2\\n\\nVector addition\\n\\nVector addition and subtraction.\\n\\nIf x and y are vectors of the same size, x+y and\\n\\nx-y give their sum and diﬀerence, respectively.\\n\\njulia>\\n\\n[\\n\\n0,\\n\\n7,\\n\\n3\\n\\n]\\n\\n+\\n\\n[\\n\\n1,\\n\\n2,\\n\\n0\\n\\n]\\n\\n#\\n\\nVector\\n\\naddition\\n\\n3-element Array{Int64,1}:\\n\\n1\\n\\n9\\n\\n3\\n\\njulia>\\n\\n[\\n\\n1,\\n\\n9 ]\\n\\n-\\n\\n[\\n\\n1,\\n\\n1]\\n\\n#\\n\\nVector\\n\\nsubtraction\\n\\n2-element Array{Int64,1}:\\n\\n0\\n\\n8\\n\\n1.3\\n\\nScalar-vector multiplication\\n\\nScalar-vector multiplication and division.\\n\\nIf a is a number and x a vector, you\\n\\ncan express the scalar-vector product either as a*x or x*a.\\n\\n(Julia actually allows\\n\\nyou to write 2.0x for 2.0*x. This is unambiguous because variable names cannot\\n\\nstart with a number.) You can carry out scalar-vector division in a similar way, as\\n\\nx/a, or the less familar looking expression a\\\\x.\\n\\njulia>\\n\\nx\\n\\n= [ 0,\\n\\n2,\\n\\n-1\\n\\n];\\n\\njulia>\\n\\n2.2 * x\\n\\n#\\n\\nScalar-vector\\n\\nmultiplication\\n\\n3-element Array{Float64,1}:\\n\\n0.0\\n\\n4.4\\n\\n-2.2\\n\\njulia>\\n\\nx\\n\\n* 2.2\\n\\n#\\n\\nScalar-vector\\n\\nmultiplication\\n\\n3-element Array{Float64,1}:\\n\\n0.0\\n\\n4.4\\n\\n-2.2\\n\\n\\x0c',\n",
       " '10\\n\\n1\\n\\nVectors\\n\\njulia> x\\n\\n/\\n\\n3\\n\\n#\\n\\nScalar-vector\\n\\ndivision\\n\\n3-element\\n\\nArray{Float64,1}:\\n\\n0.0\\n\\n0.666667\\n\\n-0.333333\\n\\njulia>\\n\\n\\x0c',\n",
       " '1.3\\n\\nScalar-vector multiplication\\n\\n11\\n\\n0.00634138\\n\\nElementwise operations with a scalar.\\n\\nElementwise operations work when one\\n\\nof\\n\\nthe arguments is a scalar,\\n\\nin which case it\\n\\nis interpreted as the scalar times a\\n\\nones vector of the appropriate dimension. Scalar-vector addition, described above,\\n\\nis a special case of this.\\nIf a is a scalar and x is a vector, then x.^a is a vector with\\nith element xa\\ni , and a.^x is a vector with elements axi . Like scalar-vector addition,\\nthe dot notation for elementwise operations is not standard mathematical notation\\n\\nand we do not use it in VMLS.\\n\\nWe can also use the period notation with a function that has a name,\\n\\nto let\\n\\nJulia know that the function should be applied elementwise.\\n\\nIn this case we add the\\n\\nperiod after\\n\\nthe function name to indicate that it should be applied elementwise.\\n\\nFor example,\\n\\nif x is a vector, we can form sin.(x) to apply the sine function to\\n\\neach element of x.\\n\\nThe equality test operator == (and other relational operators like <, >=) can be\\n\\nmade to work elementwise by preceding it with a period: x==y tells us whether or\\n\\nnot x and y are the same vector; x.==y is a vector whose entries tell us whether\\n\\nthe corresponding entries in x and y are the same.\\n\\njulia>\\n\\njulia>\\n\\nw\\n\\nw\\n\\nfalse\\n\\n= [1,2,2];\\n\\nz\\n\\n=\\n\\n[1,2,3];\\n\\n==\\n\\nz\\n\\njulia>\\n\\nw\\n\\n.== z\\n\\n3-element BitArray{1}:\\n\\ntrue\\n\\ntrue\\n\\nfalse\\n\\nYou can combine this with Julia’s slicing to extract the subvector of entries that\\n\\nsatisfy some logical condition. For example x[abs.(x).>1] gives the subvector of\\n\\nx consisting of the entries larger than one in magnitude.\\n\\njulia>\\n\\nx\\n\\n= [1.1,\\n\\n.5,\\n\\n-1.5,\\n\\n-0.3]\\n\\n4-element Array{Float64,1}:\\n\\n1.1\\n\\n0.5\\n\\n-1.5\\n\\n-0.3\\n\\njulia>\\n\\nx[abs.(x)\\n\\n.>\\n\\n1]\\n\\n2-element Array{Float64,1}:\\n\\n1.1\\n\\n-1.5\\n\\nDot notation works with assignment too, allowing you to assign multiple entries\\n\\nof a vector to a scalar value. For example:\\n\\n\\x0c',\n",
       " '12\\n\\n1\\n\\nVectors\\n\\njulia> x\\n\\n=\\n\\nrand(4)\\n\\n4-element\\n\\nArray{Float64,1}:\\n\\n0.4735758513909343\\n\\n0.3554729725184458\\n\\n0.13775623085957855\\n\\n0.9227200780245117\\n\\njulia> x[1:2]\\n\\n=\\n\\n[-1,1];\\n\\njulia> x\\n\\n4-element\\n\\nArray{Float64,1}:\\n\\n-1.0\\n\\n1.0\\n\\n0.13775623085957855\\n\\n0.9227200780245117\\n\\njulia>\\n\\nx[2:3]\\n\\n.=\\n\\n1.3;\\n\\njulia>\\n\\nx\\n\\n4-element\\n\\nArray{Float64,1}:\\n\\n-1.0\\n\\n1.3\\n\\n1.3\\n\\n3.0\\n\\nLinear\\n\\ncombination.\\n\\nYou can form a linear\\n\\ncombination is Julia using scalar-\\n\\nvector multiplication and addition.\\n\\njulia> a\\n\\n=\\n\\n[ 1,\\n\\n2\\n\\n];\\n\\nb\\n\\n=\\n\\njulia> alpha =\\n\\n-0.5;\\n\\nbeta\\n\\n[\\n\\n=\\n\\n3,\\n\\n4\\n\\n];\\n\\n1.5;\\n\\njulia> c\\n\\n=\\n\\nalpha*a\\n\\n+\\n\\nbeta*b\\n\\n2-element\\n\\nArray{Float64,1}:\\n\\n4.0\\n\\n5.0\\n\\nTo illustrate some additional Julia syntax, we create a function that\\n\\ntakes a\\n\\nlist of\\n\\ncoeﬃcients and a list of vectors as\\n\\nits arguments, and returns\\n\\nthe\\n\\nlinear\\n\\ncombination. The lists can be represented by tuples or arrays.\\n\\njulia> function\\n\\nlincomb(coeff,\\n\\nvectors)\\n\\nn\\n\\na\\n\\n= length(vectors[1])\\n\\n#\\n\\nLength\\n\\nof\\n\\nvectors\\n\\n= zeros(n);\\n\\nfor i\\n\\n= 1:length(vectors)\\n\\na\\n\\n= a\\n\\n+\\n\\ncoeff[i]\\n\\n*\\n\\nvectors[i];\\n\\nend\\n\\nreturn\\n\\na\\n\\n\\x0c',\n",
       " '1.3\\n\\nScalar-vector multiplication\\n\\n13\\n\\nend\\n\\nlincomb (generic\\n\\nfunction\\n\\nwith\\n\\n1\\n\\nmethod)\\n\\njulia> lincomb(\\n\\n(\\n\\n-0.5,\\n\\n1.5),\\n\\n(\\n\\n[1,\\n\\n2],\\n\\n[\\n\\n3,\\n\\n4]) )\\n\\n2-element\\n\\nArray{Float64,1}:\\n\\n4.0\\n\\n5.0\\n\\nA more concise deﬁnition of the function is as follows.\\n\\njulia>\\n\\nfunction\\n\\nlincomb(coeff,\\n\\nvectors)\\n\\nreturn\\n\\nsum( coeff[i]\\n\\n*\\n\\nvectors[i]\\n\\nfor\\n\\ni\\n\\n=\\n\\n1:length(vectors)\\n\\n)\\n\\nend\\n\\nChecking properties.\\n\\nLet’s check the distributive property\\n\\nβ (a + b) = β a + β b,\\n\\nwhich holds\\n\\nfor any two n-vectors a and b, and any scalar β . We’ll do this\\n\\nfor\\n\\nn = 3, and randomly generated a,\\n\\nb, and β .\\n\\n(This computation does not\\n\\nshow\\n\\nthat the property always holds;\\n\\nit only shows that it holds for the speciﬁc vectors\\n\\nchosen. But it’s good to be skeptical and check identities with random arguments.)\\n\\nWe use the lincomb function we just deﬁned.\\n\\njulia>\\n\\na\\n\\n= rand(3)\\n\\n3-element Array{Float64,1}:\\n\\n0.55304\\n\\n0.55801\\n\\n0.0299682\\n\\njulia>\\n\\nb\\n\\n= rand(3);\\n\\n3-element Array{Float64,1}:\\n\\n0.796619\\n\\n0.578865\\n\\n0.219901\\n\\njulia>\\n\\nbeta\\n\\n= randn()\\n\\n#\\n\\nGenerates\\n\\na\\n\\nrandom\\n\\nscalar\\n\\nbeta =\\n\\nrandn()\\n\\n-0.17081925677011056\\n\\njulia>\\n\\nlhs = beta*(a+b)\\n\\n3-element Array{Float64,1}:\\n\\n-0.230548\\n\\n-0.1942\\n\\n-0.0426825\\n\\njulia>\\n\\nrhs = beta*a\\n\\n+\\n\\nbeta*b\\n\\n3-element Array{Float64,1}:\\n\\n\\x0c',\n",
       " \"14\\n\\n1\\n\\nVectors\\n\\n-0.230548\\n\\n-0.1942\\n\\n-0.0426825\\n\\nAlthough the two vectors lhs and rhs are displayed as the same, they might not be\\n\\nexactly the same, due to very small round-oﬀ errors in ﬂoating point computations.\\n\\nWhen we check an identity using random numbers, we can expect\\n\\nthat\\n\\nthe left-\\n\\nhand and right-hand sides of the identity are not exactly the same, but very close\\n\\nto each other.\\n\\n1.4\\n\\nInner product\\n\\nInner product.\\n\\nThe inner product of n-vectors x and y is denoted as xT y .\\nJulia, the inner product of x and y is denoted as x'*y.\\n\\nIn\\n\\njulia> x\\n\\n=\\n\\n[ -1,\\n\\n2,\\n\\n2\\n\\n[ 1,\\n\\n0,\\n\\n-3\\n\\njulia> y\\n=\\njulia> x'*y\\n\\n-7\\n\\n];\\n\\n];\\n\\nNet\\n\\npresent\\n\\nvalue.\\n\\nAs\\n\\nan example,\\n\\nthe\\n\\nfollowing\\n\\ncode\\n\\nsnippet ﬁnds\\n\\nthe net\\n\\npresent value (NPV) of a cash ﬂow vector c, with per-period interest rate r.\\n\\njulia> c\\n\\njulia> n\\n\\njulia> r\\n\\njulia> d\\n\\n=\\n\\n=\\n\\n=\\n\\n=\\n\\n[ 0.1,\\n\\n0.1,\\n\\n0.1,\\n\\n1.1\\n\\n];\\n\\n#\\n\\nCash\\n\\nflow\\n\\nvector\\n\\nlength(c);\\n\\n0.05;\\n\\n#\\n\\n5%\\n\\nper-period\\n\\ninterest\\n\\nrate\\n\\n(1+r)\\n\\n.^\\n\\n-(0:n-1)\\n\\n4-element\\n\\nArray{Float64,1}:\\n\\n1.0\\n\\n0.952381\\n\\n0.907029\\n\\n0.863838\\n\\njulia> NPV\\n\\n= c'*d\\n\\n1.236162401468524\\n\\nIn the fourth line,\\n\\nto get the vector d we raise the scalar 1+r elementwise to the\\n\\npowers given in the array (0:n-1), which expands to [0,1,...,n-1].\\n\\nTotal\\n\\nschool-age population.\\n\\nSuppose that\\n\\nthe 100-vector x gives\\n\\ntribution of\\n\\nsome population, with xi\\ni = 1, . . . , 100.\\nThe\\ntotal number of people with age between 5 and 18 (inclu-\\n\\nthe number\\n\\nage\\n\\nof people\\n\\nof\\n\\nthe age dis-\\ni − 1,\\n\\nfor\\n\\n\\x0c\",\n",
       " \"1.5\\n\\nComplexity of vector computations\\n\\n15\\n\\nsive) is given by\\n\\nx6 + x7 + · · · + x18 + x19 .\\n\\nWe can express this as sT x, where s is the vector with entries one for i = 6, . . . , 19\\n\\nand zero otherwise.\\n\\nIn Julia, this is expressed as\\n\\njulia>\\n\\ns\\n\\n= [ zeros(5);\\n\\njulia>\\n\\nschool_age_pop\\n\\n=\\n\\nones(14);\\ns'*x\\n\\nzeros(81)\\n\\n];\\n\\nSeveral other expressions can be used to evaluate this quantity,\\n\\nfor example,\\n\\nthe\\n\\nexpression sum(x[6:19]), using the Julia function sum, which gives the sum of the\\n\\nentries of a vector.\\n\\n1.5\\n\\nComplexity of vector computations\\n\\nFloating point operations.\\n\\nFor any two numbers a and b, we have (a + b)(a − b) =\\na2 − b2 . When a computer calculates the left-hand and right-hand side, for speciﬁc\\n\\nnumbers a and b,\\n\\nthey need not be exactly the same, due to very small ﬂoating\\n\\npoint\\n\\nround-oﬀ errors.\\n\\nBut\\n\\nthey should be very nearly the\\n\\nsame.\\n\\nLet’s\\n\\nsee an\\n\\nexample of this.\\n\\njulia>\\n\\na\\n\\n= rand();\\n\\njulia>\\n\\nlhs = (a+b)\\n\\nb\\n\\n*\\n\\n=\\n\\nrand();\\n\\n(a-b)\\n\\n-0.025420920298883976\\n\\njulia>\\n\\nrhs = a^2\\n\\n-\\n\\nb^2\\n\\n-0.02542092029888398\\n\\njulia>\\n\\nlhs - rhs\\n\\n3.469446951953614e-18\\n\\nHere we see that the left-hand and right-hand sides are not exactly equal, but very\\n\\nvery close.\\n\\nComplexity.\\n\\nYou can time a Julia command by adding @time before the com-\\n\\nmand.\\nThe\\ntimer\\nis not very accurate\\nfor very small\\ntimes,\\nsay, measured in\\nmicroseconds (10−6 seconds). Also, you should run the command more than once;\\n\\nit can be a lot faster on the second or subsequent runs.\\n\\njulia>\\n\\njulia>\\n\\na\\n= randn(10^5);\\n@time a'*b\\n\\nb\\n\\n=\\n\\nrandn(10^5);\\n\\n0.002695 seconds\\n\\n(5\\n\\nallocations:\\n\\n176\\n\\nbytes)\\n\\n38.97813069037062\\na'*b\\n\\njulia>\\n\\n@time\\n\\n0.000173 seconds\\n\\n(5\\n\\nallocations:\\n\\n176\\n\\nbytes)\\n\\n\\x0c\",\n",
       " \"16\\n\\n1\\n\\nVectors\\n\\n38.97813069037062\\n\\njulia> c\\n=\\nrandn(10^6);\\njulia> @time c'*d\\n\\nd\\n\\n=\\n\\nrandn(10^6);\\n\\n0.001559\\n\\nseconds\\n\\n(5\\n\\nallocations:\\n\\n176\\n\\nbytes)\\n\\n1189.2960722446112\\njulia> @time c'*d\\n\\n0.001765\\n\\nseconds\\n\\n(5\\n\\nallocations:\\n\\n176\\n\\nbytes)\\n\\n1189.2960722446112\\n\\nThe ﬁrst\\n\\ninner product, of vectors of\\nlength 106\\n\\nlength 105 ,\\n\\ntakes around 0.00017 seconds;\\n\\nthe second, with vectors of\\n\\n(tens times bigger), product\\n\\ntakes around\\n\\n0.0018 seconds, about 10 longer. This is predicted by the complexity of the inner\\nproduct, which is 2n − 1 ﬂops.\\ndone is capable of around 2 · 106/0.001765 ﬂops per second,\\nThe computer on which the computations were\\n\\ni.e., around 1 Gﬂop/s.\\n\\nThese timings, and the estimate of computer speed, are very approximate.\\n\\nSparse vectors.\\n\\nFunctions for creating and manipulating sparse vectors are con-\\n\\ntained in the Julia package SparseArrays, so you need to install this package before\\n\\nyou can use them; see page ix.\\n\\nSparse vectors are stored as sparse arrays, i.e., arrays in which only the nonzero\\n\\nelements are stored.\\n\\nIn Julia you can create a sparse vector from lists of the indices\\n\\nand values using the sparsevec function. You can also ﬁrst create a sparse vector\\n\\nof zeros (using spzeros(n)) and then assign values to the nonzero entries. A sparse\\n\\nvector can be created from a non-sparse vector using sparse(x), which returns a\\n\\nsparse version of x. nnz(x) gives the number of nonzero elements of a sparse vector.\\n\\nSparse vectors are overloaded to work as you imagine;\\n\\nfor example, all the usual\\n\\nvector operations work, and they are automatically recast as non-sparse vectors\\n\\nwhen appropriate.\\n\\njulia> a\\n\\n=\\n\\nsparsevec(\\n\\n[\\n\\n123456,\\n\\n123457\\n\\n],\\n\\n[\\n\\n1.0, -1.0\\n\\n],\\n\\n10^6\\n\\n)\\n\\n1000000-element\\n\\nSparseVector{Float64,Int64}\\n\\nwith 2\\n\\nstored\\n\\nentries:\\n\\n[123456 ]\\n\\n=\\n\\n1.0\\n\\n[123457 ]\\n\\n=\\n\\n-1.0\\n\\njulia> length(a)\\n\\n1000000\\n\\njulia> nnz(a)\\n\\n2\\n\\njulia> b\\n\\n=\\n\\nrandn(10^6);\\n\\n#\\n\\nAn\\n\\nordinary\\n\\n(non-sparse)\\n\\nvector\\n\\njulia> @time 2*a;\\n\\n#\\n\\nComputed\\n\\nefficiently!\\n\\n0.000003\\n\\nseconds\\n\\n(7\\n\\nallocations:\\n\\n384\\n\\nbytes)\\n\\njulia> @time 2*b;\\n\\n0.003558\\nseconds\\njulia> @time a'*b;\\n\\n(6\\n\\nallocations:\\n\\n7.630\\n\\nMiB)\\n\\n# Computed\\n\\nefficiently!\\n\\n0.000003\\n\\nseconds\\n\\n(5\\n\\nallocations:\\n\\n176\\n\\nbytes)\\n\\n\\x0c\",\n",
       " \"1.5\\n\\nComplexity of vector computations\\n\\n17\\n\\njulia>\\n\\n@time b'*b;\\n\\n0.000450 seconds\\n\\n(5\\n\\nallocations:\\n\\n176\\n\\nbytes)\\n\\njulia>\\n\\n@time c\\n\\n=\\n\\na\\n\\n+\\n\\nb;\\n\\n0.002085 seconds\\n\\n(6\\n\\nallocations:\\n\\n7.630\\n\\nMiB)\\n\\nIn the last line, the sparse vector a is automatically converted to an ordinary vector\\n\\n(array) so it can be added to the random vector; the result is a (non-sparse) vector\\nlength 106 .\\n\\nof\\n\\n\\x0c\",\n",
       " '18\\n\\n1\\n\\nVectors\\n\\n\\x0c',\n",
       " \"Chapter 2\\n\\nLinear functions\\n\\n2.1\\n\\nLinear functions\\n\\nFunctions in Julia.\\n\\nJulia provides several methods for deﬁning functions. A sim-\\nple function given by an expression such as f (x) = x1 + x2 − x2\\n4 can be deﬁned in\\na single line.\\n\\njulia>\\n\\nf(x)\\n\\n= x[1]\\n\\n+\\n\\nx[2]\\n\\n-\\n\\nx[4]^2\\n\\nf (generic function\\n\\nwith\\n\\n1\\n\\nmethod)\\n\\njulia>\\n\\nf([-1,0,1,2])\\n\\n-5\\n\\nSince the function deﬁnition refers to the ﬁrst, second, and fourth elements of the\\n\\nargument x, these have to be deﬁned when you call or evaluate f(x); you’ll get an\\n\\nerror if,\\n\\nfor example, x has dimension 3 or is a scalar.\\n\\nSuperposition.\\n\\nSuppose a is an n-vector.\\n\\nThe\\n\\nfunction f (x) = aT x is\\n\\nlinear,\\n\\nwhich means that for any n-vectors x and y , and any scalars α and β , the super-\\n\\nposition equality\\n\\nf (αx + β y) = αf (x) + β f (y)\\n\\nholds. Superposition says that evaluating f at a linear combination of two vectors\\n\\nis the same forming the linear combination of f evaluated at the two vectors.\\n\\nLet’s deﬁne the inner product\\n\\nfunction f\\n\\nfor a speciﬁc value of a, and then\\n\\nverify superposition in Julia for speciﬁc values of x, y , α, and β .\\n\\n(This check does\\n\\nnot show that the function is linear.\\n\\nIt simply checks that superposition holds for\\n\\nthese speciﬁc values.)\\n\\njulia>\\n\\na\\n\\njulia>\\n\\n= [-2,\\n0,\\n= a'*x\\n\\nf(x)\\n\\n1,\\n\\n-3];\\n\\n#\\n\\nInner product\\n\\nfunction\\n\\nf (generic\\n\\nfunction\\n\\nwith\\n\\n1\\n\\nmethod)\\n\\njulia> x\\n\\n=\\n\\n[2,\\n\\n2,\\n\\n-1,\\n\\n1];\\n\\ny=\\n\\n[0,\\n\\n1,\\n\\n-1,\\n\\n0];\\n\\n\\x0c\",\n",
       " \"20\\n\\n2\\n\\nLinear functions\\n\\njulia> alpha =\\n\\n1.5;\\n\\nbeta\\n\\n=\\n\\n-3.7;\\n\\njulia> lhs\\n\\n= f(alpha\\n\\n*\\n\\nx\\n\\n+\\n\\nbeta\\n\\n*\\n\\ny)\\n\\n-8.3\\n\\njulia> rhs\\n\\n= alpha\\n\\n*\\n\\nf(x)\\n\\n+\\n\\nbeta\\n\\n*\\n\\nf(y)\\n\\n-8.3\\n\\nFor the function f (x) = aT x, we have f (e3 ) = a3 . Let’s check that this holds in\\nour example.\\n\\njulia> e3\\n\\n= [0,\\n\\n0,\\n\\n1,\\n\\n0];\\n\\njulia> f(e3)\\n\\n1.0\\n\\nExamples.\\n\\nLet’s deﬁne the average function in Julia, and check its value for a\\n\\nspeciﬁc vector.\\n\\n(Julia’s Statistics package contains the average function, which\\n\\nis called mean.)\\n\\njulia> avg(x)\\n\\n=\\n\\n(ones(length(x))\\n\\n/\\n\\nlength(x))'*x;\\n\\njulia> x\\n\\n=\\n\\n[1,\\n\\n-3,\\n\\n2,\\n\\n-1];\\n\\njulia> avg(x)\\n\\n-0.25\\n\\nThe average function can be implemented more concisely as sum(x)/length(x).\\n\\nThe avg function is part of the VMLS package; once you install and then add this\\n\\npackage, you can use the avg function.\\n\\n2.2\\n\\nTaylor approximation\\n\\nTaylor approximation.\\n\\nRn → R, at the point z ,\\n\\nThe (ﬁrst-order) Taylor approximation of a function f :\\n\\nis the aﬃne function of x given by\\n\\nˆf (x) = f (z ) + ∇f (z )T (x − z ).\\n\\nFor x near z , ˆf (x) is very close to f (x). Let’s try a numerical example (see page 36)\\n\\nusing Julia.\\n\\njulia> f(x) =\\n\\nx[1]\\n\\n+\\n\\nexp(x[2]-x[1]);\\n\\n#\\n\\nA\\n\\nfunction\\n\\njulia> #\\n\\nAnd its\\n\\ngradient\\n\\njulia> grad_f(z)\\n\\n=\\n\\n[1-exp(z[2]-z[1]),\\n\\nexp(z[2]-z[1])];\\n\\njulia> z\\n\\n=\\n\\n[1,\\n\\n2];\\n\\njulia> #\\n\\nTaylor\\n\\njulia> f_hat(x)\\n\\n=\\n\\napproximation\\nat\\nz\\ngrad_f(z)'*(x-z);\\n\\nf(z)\\n\\n+\\n\\n\\x0c\",\n",
       " '2.3\\n\\nRegression model\\n\\n21\\n\\njulia>\\n\\n#\\n\\nLet’s\\n\\ncompare\\n\\nf\\n\\nand\\n\\nf_hat\\n\\nfor\\n\\nsome\\n\\nspecific\\n\\nx’s\\n\\njulia>\\n\\nf([1,2]),\\n\\nf_hat([1,2])\\n\\n(3.718281828459045,\\n\\n3.718281828459045)\\n\\njulia>\\n\\nf([0.96,1.98]),\\n\\nf_hat([0.96,1.98])\\n\\n(3.733194763964298,\\n\\n3.732647465028226)\\n\\njulia>\\n\\nf([1.10,2.11]),\\n\\nf_hat([1.10,2.11])\\n\\n(3.845601015016916,\\n\\n3.845464646743635)\\n\\n2.3\\n\\nRegression model\\n\\nRegression model.\\n\\nThe\\nf (x) = xT β + v , where\\n\\nregression model\\n\\nis\\n\\nthe\\n\\naﬃne\\n\\nfunction of x given by\\n\\nthe n-vector β and the\\n\\nscalar v are\\n\\nthe parameters\\n\\nin\\n\\nthe model.\\n\\nThe regression model\\n\\nis used to guess or approximate a real or ob-\\n\\nserved value of the number y that is associated with x.\\n\\n(We’ll see later how to ﬁnd\\n\\nthe parameters in a regression model, using data.)\\n\\nLet’s deﬁne the regression model\\n\\nfor house sale prices described on page 39 of\\n\\nVMLS, and compare its prediction to the true house sale price y for a few values\\n\\nof x.\\n\\njulia>\\n\\n#\\n\\nParameters\\n\\nin\\n\\nregression\\n\\nmodel\\n\\njulia>\\n\\nbeta\\n\\n= [148.73,\\n-18.85];\\nx\\'*beta\\n\\nv\\n\\n=\\n\\n54.40;\\n\\njulia>\\n\\ny_hat(x)\\n\\n=\\n\\n+\\n\\nv;\\n\\njulia>\\n\\n#\\n\\nEvaluate\\n\\nregression\\n\\nmodel\\n\\nprediction\\n\\njulia>\\n\\nx\\n\\n= [0.846,\\n\\n1];\\n\\ny\\n\\n=\\n\\n115;\\n\\njulia>\\n\\ny_hat(x),\\n\\ny\\n\\n(161.37557999999999,\\n\\n115)\\n\\njulia>\\n\\nx\\n\\n= [1.324,2];\\n\\ny\\n\\n=\\n\\n234.50;\\n\\njulia>\\n\\ny_hat(x),\\n\\ny\\n\\n(213.61852000000002,\\n\\n234.5)\\n\\nOur ﬁrst prediction is pretty bad; our second one is better.\\n\\nA scatter plot of predicted and actual house prices (Figure 2.4 of VMLS) can\\n\\nbe generated as follows. We use the VMLS function house_sales_data to obtain\\n\\nthe vectors price, area, beds (see appendix A).\\n\\njulia>\\n\\nD\\n\\n= house_sales_data();\\n\\njulia>\\n\\nprice =\\n\\nD[\"price\"]\\n\\n774-element\\n\\nArray{Float64,1}:\\n\\n94.905\\n\\n98.937\\n\\n100.309\\n\\n\\x0c',\n",
       " '22\\n\\n2\\n\\nLinear functions\\n\\n106.25\\n\\n107.502\\n\\n108.75\\n\\n110.7\\n\\n113.263\\n\\n116.25\\n\\n120.0\\n...\\n\\n229.027\\n\\n229.5\\n\\n230.0\\n\\n230.0\\n\\n232.425\\n\\n234.0\\n\\n235.0\\n\\n235.301\\n\\n235.738\\n\\njulia> area =\\n\\nD[\"area\"]\\n\\n774-element Array{Float64,1}:\\n\\n0.941\\n\\n1.146\\n\\n0.909\\n\\n1.289\\n\\n1.02\\n\\n1.022\\n\\n1.134\\n\\n0.844\\n\\n0.795\\n\\n0.588\\n...\\n\\n1.358\\n\\n1.329\\n\\n1.715\\n\\n1.262\\n\\n2.28\\n\\n1.477\\n\\n1.216\\n\\n1.685\\n\\n1.362\\n\\njulia> beds =\\n\\nD[\"beds\"]\\n\\n774-element Array{Int64,1}:\\n\\n\\x0c',\n",
       " '2.3\\n\\nRegression model\\n\\n23\\n\\n2\\n\\n3\\n\\n3\\n\\n3\\n\\n3\\n\\n2\\n\\n2\\n\\n2\\n\\n2\\n\\n2\\n...\\n\\n3\\n\\n4\\n\\n4\\n\\n3\\n\\n4\\n\\n3\\n\\n3\\n\\n4\\n\\n3\\n\\njulia>\\n\\nv\\n\\n= 54.4017;\\n\\njulia>\\n\\nbeta\\n\\n= [\\n\\n147.7251,\\n\\n-18.8534\\n\\n];\\n\\njulia>\\n\\npredicted\\n\\n=\\n\\nv\\n\\n.+\\n\\nbeta[1]\\n\\n*\\n\\narea\\n\\n+\\n\\nbeta[2]\\n\\n*\\n\\nbeds;\\n\\njulia>\\n\\nusing Plots\\n\\njulia>\\n\\nscatter(price,\\n\\npredicted,\\n\\nlims\\n\\n=\\n\\n(0,800));\\n\\njulia>\\n\\nplot!([0,\\n\\n800],\\n\\n[0,\\n\\n800],\\n\\nlinestyle\\n\\n=\\n\\n:dash);\\n\\njulia>\\n\\n#\\n\\nmake axes\\n\\nequal\\n\\nand\\n\\nadd\\n\\nlabels\\n\\njulia>\\n\\nplot!(xlims\\n\\n=\\n\\n(0,800),\\n\\nylims\\n\\n=\\n\\n(0,800),\\n\\nsize\\n\\n=\\n\\n(500,500));\\n\\njulia>\\n\\nplot!(xlabel\\n\\n=\\n\\n\"Actual\\n\\nprice\",\\n\\nylabel\\n\\n=\\n\\n\"Predicted\\n\\nprice\");\\n\\n\\x0c',\n",
       " '24\\n\\n2\\n\\nLinear functions\\n\\nFigure 2.1 Scatter plot of actual and predicted sale prices\\n\\nfor 774 houses\\n\\nsold in Sacramento during a ﬁve-day period.\\n\\n02004006008000200400600800Actual pricePredicted price\\x0c',\n",
       " \"Chapter 3\\n\\nNorm and distance\\n\\n3.1\\n\\nNorm\\n\\nNorm.\\n\\nThe norm (cid:107)x(cid:107)\\n\\nis written in Julia\\n\\nas\\n\\nnorm(x).\\n\\n(It\\n\\ncan be\\n\\nevaluated\\n\\nseveral other ways\\n\\ntoo.)\\n\\nThe norm function is\\n\\ncontained in the Julia package\\n\\nLinearAlgebra, so you must install and then add this package to use it; see page ix.\\n\\njulia>\\n\\nx\\n\\n= [ 2,\\n\\n-1,\\n\\n2\\n\\n];\\n\\njulia>\\n\\nnorm(x)\\n\\n3.0\\n\\njulia>\\n\\n3.0\\n\\nsqrt(x'*x))\\n\\njulia>\\n\\nsqrt(sum(x.^2))\\n\\n3.0\\n\\nTriangle inequality.\\n\\nLet’s check the triangle inequality, (cid:107)x + y(cid:107) ≤ (cid:107)x(cid:107) + (cid:107)y(cid:107),\\n\\nfor\\n\\nsome speciﬁc values of x and y .\\n\\njulia>\\n\\nx\\n\\n= randn(10);\\n\\ny\\n\\n=\\n\\nrandn(10);\\n\\njulia>\\n\\nlhs = norm(x+y)\\n\\n3.5830436972032644\\n\\njulia>\\n\\nrhs = norm(x)\\n\\n+\\n\\nnorm(y)\\n\\n4.937368598697242\\n\\nRMS value.\\n\\nThe RMS value of a vector x is rms(x) = (cid:107)x(cid:107)/\\n\\n√\\n\\nn.\\n\\nIn Julia, this is\\n\\nexpressed as norm(x)/sqrt(length(x)).\\n\\n(The VMLS package contains\\n\\nthis\\n\\nfunc-\\n\\ntion, so you can use it once you’ve installed this package.)\\n\\nLet’s deﬁne a vector (which represents a signal,\\n\\ni.e., the value of some quantity\\n\\nat uniformly space time instances), and ﬁnd its RMS value. The following code\\n\\n\\x0c\",\n",
       " '26\\n\\n3\\n\\nNorm and distance\\n\\nFigure 3.1 A signal x. The horizontal\\nand avg(x) − rms(x).\\n\\nlines show avg(x) + rms(x), avg(x),\\n\\nplots\\n\\nthe signal,\\n\\nits average value, and two constant\\n\\nsignals at avg(x) ± rms(x)\\n\\n(Figure 3.1).\\n\\njulia> rms(x)\\n\\n=\\n\\nnorm(x)\\n\\n/\\n\\nsqrt(length(x));\\n\\njulia> t\\n\\njulia> x\\n\\n=\\n\\n=\\n\\n0:0.01:1;\\n\\n#\\n\\nList\\n\\nof\\n\\ntimes\\n\\ncos.(8*t)\\n\\n-\\n\\n2*sin.(11*t);\\n\\njulia> avg(x)\\n\\n-0.04252943783238692\\n\\njulia> rms(x)\\n\\n1.0837556422598\\n\\njulia> using Plots\\n\\njulia> plot(t,\\n\\nx)\\n\\njulia> plot!(t,\\n\\navg(x)*ones(length(x)))\\n\\njulia> plot!(t,\\n\\n(avg(x)+rms(x))*ones(length(x)), color\\n\\njulia> plot!(t,\\n\\n(avg(x)-rms(x))*ones(length(x)), color\\n\\n=\\n\\n=\\n\\n:green)\\n\\n:green)\\n\\njulia> plot!(legend\\n\\n=\\n\\nfalse)\\n\\nChebyshev inequality.\\n\\nof an n-vector x that have absolute value at\\nn rms(x)2 /a2 .\\n\\nthis number\\n\\nIf\\n\\nThe Chebyshev inequality states that the number of entries\\nleast a is no more than (cid:107)x(cid:107)2/a2 =\\n\\nis,\\n\\nsay, 12.15, we can conclude that no more that\\n\\n12 entries have absolute value at least a, since the number of entries is an integer.\\nSo the Chebyshev bound can be improved to be ﬂoor((cid:107)x(cid:107)2/a), where ﬂoor(u)\\n\\nis\\n\\nthe integer part of a positive number. Let’s deﬁne a function with the Chebyshev\\n\\nbound, including the ﬂoor function improvement, and apply the bound to the signal\\n\\nfound above,\\n\\nfor a speciﬁc value of a.\\n\\n0.000.250.500.751.00-1012\\x0c',\n",
       " '3.2\\n\\nDistance\\n\\n27\\n\\njulia>\\n\\n#\\n\\nDefine\\n\\nChebyshev\\n\\nbound\\n\\nfunction\\n\\njulia>\\n\\ncheb_bound(x,a)\\n\\n=\\n\\nfloor(norm(x)^2/a);\\n\\njulia>\\n\\na\\n\\n= 1.5;\\n\\njulia>\\n\\ncheb_bound(x,a)\\n\\n79.0\\n\\njulia>\\n\\n#\\n\\nNumber\\n\\nof\\n\\nentries\\n\\nof\\n\\nx\\n\\nwith\\n\\n|x_i|\\n\\n>=\\n\\na\\n\\njulia>\\n\\nsum(abs.(x)\\n\\n.>=\\n\\na)\\n\\n20\\n\\nIn the last line, the expression abs.(x)\\n\\n.>=\\n\\na creates an array with entries that\\n\\nare Boolean, i.e., true or false, depending on whether the corresponding entry of\\n\\nx satisﬁes the inequality. When we sum the vector of Booleans, they are automat-\\n\\nically converted to (re-cast as) the numbers 1 and 0, respectively.\\n\\n3.2\\n\\nDistance\\n\\nDistance.\\n\\nThe distance between two vectors\\n\\nis dist(x, y) = (cid:107)x − y(cid:107).\\n\\nThis\\n\\nis\\n\\nwritten in Julia as norm(x-y).\\n\\nLet’s ﬁnd the distance between the pairs of\\n\\nthe\\n\\nthree vectors u, v , and w from page 49 of VMLS.\\n\\njulia>\\n\\nu\\n\\n= [1.8,\\n\\n2.0,\\n\\n-3.7,\\n\\n4.7];\\n\\njulia>\\n\\nv\\n\\n= [0.6,\\n\\n2.1,\\n\\n1.9,\\n\\n-1.4];\\n\\njulia>\\n\\nw\\n\\n= [2.0,\\n\\n1.9,\\n\\n-4.0,\\n\\n4.6];\\n\\njulia>\\n\\nnorm(u-v),\\n\\nnorm(u-w),\\n\\nnorm(v-w)\\n\\n(8.36779540858881,\\n\\n0.3872983346207417,\\n\\n8.532877591996735)\\n\\nWe can see that u and w are much closer to each other than u and v , or v and w.\\n\\nNearest neighbor. We deﬁne a function that calculates the nearest neighbor of a\\n\\nvector in a list of vectors, and try it on the points in Figure 3.3 of VMLS.\\n\\njulia>\\n\\nnearest_neighbor(x,z)\\n\\n=\\n\\nz[\\n\\nargmin([norm(x-y)\\n\\nfor\\n\\ny\\n\\nin\\n\\nz])\\n\\n];\\n\\njulia>\\n\\nz\\n\\n= ( [2,1],\\n\\n[7,2],\\n\\n[5.5,4],\\n\\n[4,8],\\n\\n[1,5],\\n\\n[9,6]\\n\\n);\\n\\njulia>\\n\\nnearest_neighbor([5,6],\\n\\nz)\\n\\n2-element Array{Float64,1}:\\n\\n5.5\\n\\n4.0\\n\\njulia>\\n\\nnearest_neighbor([3,3],\\n\\nz)\\n\\n2-element Array{Int64,1}:\\n\\n2\\n\\n1\\n\\n\\x0c',\n",
       " '28\\n\\n3\\n\\nNorm and distance\\n\\nOn the ﬁrst line, the expression [norm(x-y)\\n\\nfor\\n\\ny\\n\\nin\\n\\nz] uses a convenient con-\\n\\nstruction in Julia. Here z is a list of vectors, and the expression expands to an array\\n\\nwith elements norm(x-z[1]), norm(x-z[2]),\\n\\n. . . . The function argmin applied to\\n\\nthis array returns the index of the smallest element.\\n\\nDe-meaning a vector. We\\n\\nrefer\\n\\nto the vector x − avg(x)1 as\\n\\nthe de-meaned\\n\\nversion of x.\\n\\njulia> de_mean(x)\\n\\n=\\n\\nx\\n\\n.-\\n\\navg(x);\\n\\n#\\n\\nDefine\\n\\nde-mean\\n\\nfunction\\n\\njulia> x\\n\\n=\\n\\n[1,\\n\\n-2.2,\\n\\n3];\\n\\njulia> avg(x)\\n\\n0.6\\n\\njulia> x_tilde\\n\\n=\\n\\nde_mean(x)\\n\\n3-element\\n\\nArray{Float64,1}:\\n\\n0.4\\n\\n-2.8\\n\\n2.4\\n\\njulia> avg(x_tilde)\\n\\n-1.4802973661668753e-16\\n\\n(The mean of ˜x is very very close to zero.)\\n\\n3.3\\n\\nStandard deviation\\n\\nStandard deviation. We\\n\\ndeﬁnition of\\n\\nthe\\n\\nstandard deviation of a vector,\\n\\ncan deﬁne a function that\\ncorresponds\\nto the VMLS\\nstd(x) = (cid:107)x − avg(x)1(cid:107)/\\n\\n√\\n\\nn,\\n\\nwhere n is the length of the vector.\\n\\njulia> x\\n\\n=\\n\\nrand(100);\\n\\njulia> #\\n\\nVMLS\\n\\ndefinition\\n\\nof\\n\\nstd\\n\\njulia> stdev(x)\\n\\n=\\n\\nnorm(x-avg(x))/sqrt(length(x));\\n\\njulia> stdev(x)\\n\\n0.292205696281305\\n\\nThis function is in the VMLS package, so you can use it once you’ve installed this\\n\\npackage.\\n(Julia’s Statistics package has a similar function, std(x), which com-\\nputes the value (cid:107)x − avg(x)1(cid:107)/\\nn − 1, where n is the length of x.)\\n\\n√\\n\\nReturn and risk. We evaluate the mean return and risk (measured by standard\\n\\ndeviation) of the four time series Figure 3.4 of VMLS.\\n\\njulia> a\\n\\n=\\n\\nones(10);\\n\\njulia> avg(a),\\n\\nstdev(a)\\n\\n\\x0c',\n",
       " \"3.4\\n\\nAngle\\n\\n29\\n\\n(1.0, 0.0)\\n\\njulia>\\n\\nb\\n\\n= [ 5,\\n\\n1,\\n\\n-2,\\n\\n3,\\n\\n6,\\n\\n3,\\n\\n-1,\\n\\n3,\\n\\n4,\\n\\n1\\n\\n];\\n\\njulia>\\n\\navg(b),\\n\\nstdev(b)\\n\\n(2.3, 2.41039415863879)\\n\\njulia>\\n\\nc\\n\\n= [ 5,\\n\\n7,\\n\\n-2,\\n\\n2,\\n\\n-3,\\n\\n1,\\n\\n-1,\\n\\n2,\\n\\n7,\\n\\n8\\n\\n];\\n\\njulia>\\n\\navg(c),\\n\\nstdev(c)\\n\\n(2.6, 3.773592452822641)\\n\\njulia>\\n\\nd\\n\\n= [ -1,\\n\\n-3,\\n\\n-4,\\n\\n-3,\\n\\n7,\\n\\n-1,\\n\\n0,\\n\\n3,\\n\\n9,\\n\\n5\\n\\n];\\n\\njulia>\\n\\navg(d),\\n\\nstdev(d)\\n\\n(1.2, 4.308131845707603)\\n\\nStandardizing a vector.\\n\\nIf a vector x isn’t constant (i.e., at least two of its entries\\n\\nare diﬀerent), we can standardize it, by subtracting its mean and dividing by its\\n\\nstandard deviation. The resulting standardized vector has mean value zero and\\n\\nRMS value one.\\n\\nIts entries are called z -scores. We’ll deﬁne a standardize function,\\n\\nand then check it with a random vector.\\n\\njulia>\\n\\nfunction\\n\\nstandardize(x)\\n\\nx_tilde =\\n\\nx .-\\n\\navg(x)\\n\\n#\\n\\nDe-meaned\\n\\nvector\\n\\nreturn\\n\\nx_tilde/rms(x_tilde)\\n\\nend\\n\\njulia> x\\n\\n=\\n\\nrand(100);\\n\\njulia> avg(x),\\n\\nrms(x)\\n\\n(0.510027255229345,\\n\\n0.5883938729563185)\\n\\njulia> z\\n\\n=\\n\\nstandardize(x);\\n\\njulia> avg(z),\\n\\nrms(z)\\n\\n(1.965094753586527e-16,\\n\\n1.0)\\n\\nThe mean or average value of the standarized vector z is very nearly zero.\\n\\n3.4\\n\\nAngle\\n\\nAngle.\\n\\nLet’s deﬁne a function that computes the angle between two vectors. We\\n\\nwill call\\n\\nit ang because Julia already includes a function angle (for the phase angle\\n\\nof a complex number).\\n\\njulia>\\n\\n#\\n\\nDefine\\n\\nangle\\nfunction,\\nwhich\\nreturns\\nradians\\nacos(x'*y/(norm(x)*norm(y)));\\n\\njulia>\\n\\nang(x,y)\\n\\n=\\n\\njulia> a\\n\\n=\\n\\n[1,2,-1];\\n\\nb=[2,0,-3];\\n\\njulia> ang(a,b)\\n\\n0.9689825515916383\\n\\n\\x0c\",\n",
       " \"30\\n\\n3\\n\\nNorm and distance\\n\\njulia> ang(a,b)*(360/(2*pi))\\n\\n#\\n\\nGet\\n\\nangle\\n\\nin\\n\\ndegrees\\n\\n55.51861062801842\\n\\nCorrelation coeﬃcient.\\n\\nThe correlation coeﬃcient between two vectors a and b\\n\\n(with nonzero standard deviation) is deﬁned as\\n\\nρ =\\n\\n˜aT ˜b\\n(cid:107)˜a(cid:107)(cid:107)˜b(cid:107) ,\\n\\nwhere ˜a and ˜b are the de-meaned versions of a and b,\\n\\nrespectively. There is no\\n\\nbuilt-in function for correlation, so we can deﬁne one. We use function to calculate\\n\\nthe correlation coeﬃcients of the three pairs of vectors in Figure 3.8 in VMLS.\\n\\njulia> function\\n\\ncorrel_coef(a,b)\\n\\na_tilde =\\n\\na\\n\\n.-\\n\\navg(a)\\n\\nb_tilde =\\nb\\n.-\\navg(b)\\n(a_tilde'*b_tilde)/(norm(a_tilde)*norm(b_tilde))\\n\\nreturn\\n\\nend\\n\\njulia>\\n\\na = [4.4,\\n\\n9.4,\\n\\n15.4,\\n\\n12.4,\\n\\n10.4,\\n\\n1.4,\\n\\n-4.6,\\n\\n-5.6,\\n\\n-0.6,\\n\\n7.4];\\n\\njulia>\\n\\nb = [6.2,\\n\\n11.2,\\n\\n14.2,\\n\\n14.2,\\n\\n8.2,\\n\\n2.2,\\n\\n-3.8,\\n\\n-4.8,\\n\\n-1.8,\\n\\n4.2];\\n\\njulia>\\n\\ncorrel_coef(a,b)\\n\\n0.9678196342570434\\n\\njulia>\\n\\na = [4.1,\\n\\n10.1,\\n\\n15.1,\\n\\n13.1,\\n\\n7.1,\\n\\n2.1,\\n\\n-2.9,\\n\\n-5.9,\\n\\n0.1,\\n\\n7.1];\\n\\njulia>\\n\\nb = [5.5,\\n\\n-0.5,\\n\\n-4.5,\\n\\n-3.5,\\n\\n1.5,\\n\\n7.5,\\n\\n13.5,\\n\\n14.5,\\n\\n11.5,\\n\\n4.5];\\n\\njulia>\\n\\ncorrel_coef(a,b)\\n\\n-0.9875211120643734\\n\\njulia>\\n\\na = [-5.0,\\n\\n0.0,\\n\\n5.0,\\n\\n8.0,\\n\\n13.0,\\n\\n11.0,\\n\\n1.0, 6.0,\\n\\n4.0,\\n\\n7.0];\\n\\njulia>\\n\\nb = [5.8,\\n\\n0.8,\\n\\n7.8,\\n\\n9.8,\\n\\n0.8,\\n\\n11.8,\\n\\n10.8, 5.8,\\n\\n-0.2,\\n\\n-3.2];\\n\\njulia>\\n\\ncorrel_coef(a,b)\\n\\n0.004020976661367021\\n\\nThe correlation coeﬃcients of\\n\\nthe three pairs of vectors are 96.8%, −98.8%, and\\n\\n0.4%.\\n\\n3.5\\n\\nComplexity\\n\\nLet’s check that the time to compute the correlation coeﬃcient of two n-vectors is\\n\\napproximately linear in n.\\n\\njulia> x\\n\\n=\\n\\nrandn(10^6);\\n\\ny\\n\\n=\\n\\nrandn(10^6);\\n\\njulia> @time correl_coef(x,y);\\n\\n\\x0c\",\n",
       " '3.5\\n\\nComplexity\\n\\n31\\n\\n0.131375\\n\\nseconds\\n\\n(33.01\\n\\nk\\n\\nallocations:\\n\\n16.913\\n\\nMiB,\\n\\n2.86%\\n\\ngc\\n\\ntime)\\n\\njulia>\\n\\n@time correl_coef(x,y);\\n\\n0.023760\\n\\nseconds\\n\\n(9\\n\\nallocations:\\n\\n15.259\\n\\nMiB,\\n\\n31.84%\\n\\ngc\\n\\ntime)\\n\\njulia>\\n\\nx\\n\\n= randn(10^7);\\n\\ny\\n\\n=\\n\\nrandn(10^7);\\n\\njulia>\\n\\n@time correl_coef(x,y)\\n\\n0.296075 seconds\\n\\n(9\\n\\nallocations:\\n\\n152.588\\n\\nMiB,\\n\\n30.16%\\n\\ngc\\n\\ntime)\\n\\njulia>\\n\\n@time correl_coef(x,y)\\n\\n0.118979 seconds\\n\\n(9\\n\\nallocations:\\n\\n152.588\\n\\nMiB,\\n\\n21.53%\\n\\ngc\\n\\ntime)\\n\\n\\x0c',\n",
       " '32\\n\\n3\\n\\nNorm and distance\\n\\n\\x0c',\n",
       " 'Chapter 4\\n\\nClustering\\n\\n4.1\\n\\nClustering\\n\\n4.2\\n\\nA clustering objective\\n\\nIn Julia, we can store the list of vectors\\n\\nin a Julia list or\\n\\ntuple of N vectors.\\n\\nIf\\n\\nwe call this list x, we can access the ith entry (which is a vector) using x[i]. To\\n\\nspecify the clusters or group membership, we can use a list of assignments called\\n\\nassignment, where assignment[i] is the number of the group that vector x[i]\\n\\nis assigned to.\\n\\n(This\\n\\nis an integer between 1 and k .)\\n\\n(In VMLS chapter 4, we\\n\\ndescribe the assignments using a vector c or\\n\\nthe subsets Gj .) We can store the\\nk cluster representatives as a Julia list called reps, with reps[j] the j th cluster\\n\\nrepresentative.\\n\\n(In VMLS we describe the representatives as the vectors z1 , . . . , zk .)\\n\\njulia>\\n\\nJclust(x,reps,assignment)\\n\\n=\\n\\navg(\\n\\n[norm(x[i]-reps[assignment[i]])^2\\n\\nfor\\n\\ni=1:length(x)]\\n\\n)\\n\\nJclust (generic\\n\\nfunction\\n\\nwith\\n\\n1\\n\\nmethod)\\n\\njulia> x\\n\\n=\\n\\n[ [0,1],\\n\\n[1,0],\\n\\n[-1,1]\\n\\n]\\n\\n3-element\\n\\nArray{Array{Int64,1},1}:\\n\\n[0,\\n\\n[1,\\n\\n1]\\n\\n0]\\n\\n[-1, 1]\\n\\njulia> reps =\\n\\n[\\n\\n[1,1],\\n\\n[0,0]\\n\\n]\\n\\n2-element\\n\\nArray{Array{Int64,1},1}:\\n\\n[1,\\n\\n[0,\\n\\n1]\\n\\n0]\\n\\njulia> assignment\\n\\n=\\n\\n[1,2,1]\\n\\n3-element\\n\\nArray{Int64,1}:\\n\\n1\\n\\n2\\n\\n\\x0c',\n",
       " '34\\n\\n4\\n\\nClustering\\n\\n1\\n\\njulia> Jclust(x,reps,assignment)\\n\\n2.0\\n\\njulia> assignment\\n\\n=\\n\\n[1,1,2]\\n\\n1.3333333333333333\\n\\n4.3\\n\\nThe k-means algorithm\\n\\nWe write a simple Julia implementation of the k-means algorithm and apply it to\\n\\na set of points in a plane, similar to the example in Figure 4.1 of VMLS.\\n\\nWe ﬁrst create a function kmeans that can be called as\\n\\njulia> assignment,\\n\\nrepresentatives\\n\\n=\\n\\nkmeans(x,\\n\\nk)\\n\\nwhere x is an array of N vectors and k is the number of groups. The ﬁrst output\\n\\nargument\\n\\nis an array of N integers, containing the computed group assignments\\n\\n(integers from 1 to k). The second output argument is an array of k vectors, with\\n\\nthe k group representatives. We also include two optional keyword arguments, with\\n\\na limit on the number of\\n\\niterations and a tolerance used in the stopping condition.\\n\\n1\\n\\nfunction\\n\\nkmeans(x,\\n\\nk;\\n\\nmaxiters\\n\\n=\\n\\n100,\\n\\ntol\\n\\n=\\n\\n1e-5)\\n\\n2\\n\\n3\\n\\nN\\n\\n= length(x)\\n\\n4\\n\\nn\\n\\n= length(x[1])\\n\\n5\\n\\ndistances\\n\\n= zeros(N)\\n\\n#\\n\\nused\\n\\nto\\n\\nstore\\n\\nthe\\n\\ndistance\\n\\nof\\n\\neach\\n\\n6\\n\\n#\\n\\npoint\\n\\nto\\n\\nthe\\n\\nnearest\\n\\nrepresentative.\\n\\n7\\n\\nreps\\n\\n= [zeros(n)\\n\\nfor\\n\\nj=1:k]\\n\\n#\\n\\nused\\n\\nto\\n\\nstore\\n\\nrepresentatives.\\n\\n8\\n\\n9\\n\\n#\\n\\n’assignment’\\n\\nis\\n\\nan\\n\\narray\\n\\nof\\n\\nN\\n\\nintegers\\n\\nbetween\\n\\n1\\n\\nand\\n\\nk.\\n\\n10\\n\\n#\\n\\nThe\\n\\ninitial\\n\\nassignment\\n\\nis\\n\\nchosen\\n\\nrandomly.\\n\\n11\\n\\nassignment\\n\\n= [\\n\\nrand(1:k)\\n\\nfor\\n\\ni\\n\\nin\\n\\n1:N\\n\\n]\\n\\n12\\n\\n13\\n\\nJprevious\\n\\n= Inf\\n\\n#\\n\\nused\\n\\nin\\n\\nstopping\\n\\ncondition\\n\\n14\\n\\nfor iter\\n\\n=\\n\\n1:maxiters\\n\\n15\\n\\n16\\n\\n#\\n\\nCluster\\n\\nj representative\\n\\nis\\n\\naverage\\n\\nof\\n\\npoints\\n\\nin\\n\\ncluster\\n\\nj.\\n\\n17\\n\\nfor j\\n\\n= 1:k\\n\\n18\\n\\ngroup\\n\\n=\\n\\n[i for\\n\\ni=1:N\\n\\nif\\n\\nassignment[i]\\n\\n==\\n\\nj]\\n\\n19\\n\\nreps[j]\\n\\n= sum(x[group])\\n\\n/\\n\\nlength(group);\\n\\n20\\n\\nend;\\n\\n21\\n\\n22\\n\\n#\\n\\nFor\\n\\neach x[i],\\n\\nfind\\n\\ndistance\\n\\nto\\n\\nthe\\n\\nnearest\\n\\nrepresentative\\n\\n\\x0c',\n",
       " '4.3\\n\\nThe k-means algorithm\\n\\n35\\n\\n23\\n\\n#\\n\\nand\\n\\nits\\n\\ngroup\\n\\nindex.\\n\\n24\\n\\nfor i\\n\\n= 1:N\\n\\n25\\n\\n(distances[i],\\n\\nassignment[i])\\n\\n=\\n\\n26\\n\\nfindmin([norm(x[i]\\n\\n-\\n\\nreps[j])\\n\\nfor\\n\\nj\\n\\n=\\n\\n1:k])\\n\\n27\\n\\nend;\\n\\n28\\n\\n29\\n\\n#\\n\\nCompute\\n\\nclustering\\n\\nobjective.\\n\\n30\\n\\nJ\\n\\n= norm(distances)^2\\n\\n/\\n\\nN\\n\\n31\\n\\n32\\n\\n#\\n\\nShow progress\\n\\nand\\n\\nterminate\\n\\nif\\n\\nJ\\n\\nstopped\\n\\ndecreasing.\\n\\n33\\n\\nprintln(\"Iteration\\n\\n\",\\n\\niter,\\n\\n\":\\n\\nJclust\\n\\n=\\n\\n\",\\n\\nJ,\\n\\n\".\")\\n\\n34\\n\\nif iter >\\n\\n1 &&\\n\\nabs(J\\n\\n-\\n\\nJprevious)\\n\\n<\\n\\ntol\\n\\n*\\n\\nJ\\n\\n35\\n\\nreturn assignment,\\n\\nreps\\n\\n36\\n\\nend\\n\\n37\\n\\nJprevious\\n\\n= J\\n\\n38\\n\\nend\\n\\n39\\n\\n40\\n\\nend\\n\\nInitialization.\\n\\nAs discussed in VMLS (page 76), the k-means algorithm can start\\n\\nfrom a random initial choice of\\n\\nrepresentatives, or\\n\\nfrom a random assignment of\\n\\nthe points in k groups.\\n\\nIn this implementation, we use the second option (line 11).\\n\\nThe Julia function rand(1:k) picks a random number from the set 1:k,\\n\\ni.e., the\\n\\nintegers 1, . . . , k . On line 11 we create an array assignment of N elements, with\\n\\neach element chosen by calling rand(1:k).\\n\\nUpdating group representatives.\\n\\nLines 17–20 update\\n\\nthe k group representa-\\n\\ntives.\\n\\nIn line 18, we ﬁnd the indexes of\\n\\nthe points\\n\\nin cluster j and collect\\n\\nthem\\n\\nin an array group. The expression x[group] on line 19 constructs an array from\\n\\nthe\\n\\nsubset of\\n\\nelements of x indexed by group.\\n\\nThe\\n\\nfunction sum computes\\n\\nthe\\n\\nsum of the elements of the array x[group]. Dividing by the number of elements\\n\\nlength(x[group]) gives the average of the vectors in the group. The result is j th\\n\\nthe group representative. This vector is stored as the j th element in an array reps\\n\\nof\\n\\nlength N .\\n\\nUpdating group assignments.\\n\\nOn lines 24–27 we update the group assignments.\\n\\nThe Julia function findmin computes both the minimum of a sequence of numbers\\n\\nand the position of the minimum in the sequence. The result is returned as a 2-\\n\\ntuple. On lines 25–26, we apply findmin to the array of k distances of point x[i]\\n\\nto the k representatives. We store the distance to the nearest\\n\\nrepresentative in\\n\\ndistances[i], and the index of the nearest representative (i.e., the new assignment\\n\\nof point i) in assignment[i].\\n\\nClustering objective.\\n\\nOn line 30 we compute the clustering ob jective J clust (equa-\\n\\ntion (4.1) in VMLS) as the square of the RMS value of the vector of distances.\\n\\n\\x0c',\n",
       " '36\\n\\n4\\n\\nClustering\\n\\nFigure 4.1 300 points in a plane.\\n\\nConvergence. We terminate the algorithm when the improvement in the cluster-\\n\\ning ob jective becomes very small (lines 34–36).\\n\\nExample. We apply the algorithm on a randomly generated set of N = 300 points,\\n\\nshown in Figure 4.1. These points were generated as follows.\\n\\njulia> X\\n\\n=\\n\\nvcat(\\n\\n[\\n\\n0.3*randn(2)\\n\\nfor\\n\\ni\\n\\n=\\n\\n1:100\\n\\n],\\n\\n[\\n\\n[\\n\\n[1,1] +\\n\\n0.3*randn(2)\\n\\nfor\\n\\ni\\n\\n=\\n\\n1:100\\n\\n],\\n\\n[1,-1]\\n\\n+\\n\\n0.3*randn(2)\\n\\nfor\\n\\ni\\n\\n=\\n\\n1:100\\n\\n]\\n\\n)\\n\\njulia> scatter([x[1]\\n\\nfor\\n\\nx\\n\\nin\\n\\nX],\\n\\n[x[2]\\n\\nfor\\n\\nx\\n\\nin X])\\n\\njulia> plot!(legend\\n\\n=\\n\\nfalse,\\n\\ngrid\\n\\n=\\n\\nfalse,\\n\\nsize\\n\\n= (500,500),\\n\\nxlims\\n\\n= (-1.5,2.5),\\n\\nylims\\n\\n=\\n\\n(-2,2))\\n\\nOn the ﬁrst\\nline we generate\\nthree arrays of vectors.\\nEach set\\nconsists of 100\\nvectors chosen randomly around one of the three points (0, 0), (1, 1), and (1, −1).\\n\\nThe three arrays are concatenated using vcat to get an array of 300 points.\\n\\nNext, we apply our kmeans function and make a ﬁgure with the three clusters\\n\\n(Figure 4.2).\\n\\njulia> assignment,\\n\\nreps\\n\\n=\\n\\nkmeans(X,\\n\\n3)\\n\\nIteration\\n\\n1: Jclust\\n\\nIteration\\n\\n2: Jclust\\n\\nIteration\\n\\n3: Jclust\\n\\nIteration\\n\\n4: Jclust\\n\\n=\\n\\n=\\n\\n=\\n\\n=\\n\\n0.8815722022603146.\\n\\n0.24189035975341422.\\n\\n0.18259342207994636.\\n\\n0.1800980527878161.\\n\\n-1012-2-1012\\x0c',\n",
       " '4.4\\n\\nExamples\\n\\n37\\n\\nFigure 4.2 Final clustering.\\n\\nIteration 5: Jclust\\n\\nIteration 6: Jclust\\n\\nIteration 7: Jclust\\n\\n=\\n\\n=\\n\\n=\\n\\n0.17993051934500726.\\n\\n0.17988967509836415.\\n\\n0.17988967509836415.\\n\\njulia>\\n\\ngrps\\n\\n=\\n\\n[[X[i]\\n\\nfor\\n\\ni=1:N\\n\\nif\\n\\nassignment[i]\\n\\n==\\n\\njulia>\\n\\nscatter([c[1]\\n\\nfor\\n\\nc\\n\\nin\\n\\ngrps[1]],\\n\\n[c[2]\\n\\nfor\\n\\nc\\n\\nj]\\n\\nin\\n\\nfor\\n\\nj=1:k]\\n\\ngrps[1]])\\n\\njulia>\\n\\nscatter!([c[1]\\n\\nfor\\n\\njulia>\\n\\nscatter!([c[1]\\n\\nfor\\n\\nc\\n\\nc\\n\\nin\\n\\nin\\n\\ngrps[2]],\\n\\n[c[2]\\n\\nfor\\n\\ngrps[3]],\\n\\n[c[2]\\n\\nfor\\n\\nc\\n\\nc\\n\\nin\\n\\nin\\n\\ngrps[2]])\\n\\ngrps[3]])\\n\\njulia>\\n\\nplot!(legend\\n\\n=\\n\\nfalse,\\n\\ngrid\\n\\n=\\n\\nfalse,\\n\\nsize\\n\\n=\\n\\n(500,500),\\n\\nxlims\\n\\n= (-1.5,2.5),\\n\\nylims\\n\\n=\\n\\n(-2,2))\\n\\n4.4\\n\\nExamples\\n\\n4.5\\n\\nApplications\\n\\n-1012-2-1012\\x0c',\n",
       " '38\\n\\n4\\n\\nClustering\\n\\n\\x0c',\n",
       " 'Chapter 5\\n\\nLinear independence\\n\\n5.1\\n\\nLinear dependence\\n\\n5.2\\n\\nBasis\\n\\nCash ﬂow replication.\\n\\nLet’s consider cash ﬂows over 3 periods, given by 3-vectors.\\n\\nWe know from VMLS page 93 that the vectors\\n\\n\\uf8f9\\uf8fb ,\\n\\n\\uf8ee\\uf8f0 1\\n\\n0\\n\\n0\\n\\n\\uf8ee\\uf8f0\\n\\ne1 =\\n\\nl1 =\\n\\n\\uf8f9\\uf8fb ,\\n\\n\\uf8ee\\uf8f0\\n\\n1\\n−(1 + r)\\n\\n0\\n\\nl2 =\\n\\n0\\n\\n1\\n−(1 + r)\\n\\n\\uf8f9\\uf8fb\\n\\nform a basis, wher r is the (positive) per-period interest rate. The ﬁrst vector e1\\nis a single payment of $1 in period (time) t = 1. The second vector l1 is loan of $1\\nin period t = 1, paid back in period t = 2 with interest r. The third vector l2\\nis\\nloan of $1 in period t = 2, paid back in period t = 3 with interest r. Let’s use this\\nbasis to replicate the cash ﬂow c = (1, 2, −3) as\\n\\uf8f9\\uf8fb + α2\\n\\uf8f9\\uf8fb + α3\\n\\nc = α1 e1 + α2 l1 + α3 l2 = α1\\n\\n\\uf8ee\\uf8f0 1\\n\\n1\\n−(1 + r)\\n\\n\\uf8f9\\uf8fb .\\n\\n\\uf8ee\\uf8f0\\n\\n\\uf8ee\\uf8f0\\n\\n0\\n\\n0\\n\\n0\\n\\n0\\n\\n1\\n−(1 + r)\\n\\nFrom the third component we have c3 = α3 (−(1 + r)), so α3 = −c3 /(1 + r). From\\nthe second component we have\\n\\nc2 = α2 (−(1 + r)) + α3 = α2 (−(1 + r)) − c3/(1 + r),\\n\\nso α2 = −c2 /(1 + r) − c3/(1 + r)2 . Finally from c1 = α1 + α2 , we have\\n\\nα1 = c1 + c2 /(1 + r) + c3/(1 + r)2 ,\\n\\nwhich is the net present value (NPV) of the cash ﬂow c.\\n\\nLet’s check this in Julia using an interest rate of 5% per period, and the speciﬁc\\ncash ﬂow c = (1, 2, −3).\\n\\n\\x0c',\n",
       " \"40\\n\\n5\\n\\nLinear independence\\n\\njulia> r\\n\\n=\\n\\n0.05;\\n\\njulia> e1\\n\\n= [1,0,0];\\n\\nl1\\n\\n=\\n\\n[1,-(1+r),0];\\n\\nl2\\n\\n=\\n\\n[0,1,-(1+r)];\\n\\njulia> c\\n\\n=\\n\\n[1,2,-3];\\n\\njulia> #\\n\\nCoefficients\\n\\nof\\n\\nexpansion\\n\\njulia> alpha3\\n\\n=\\n\\n-c[3]/(1+r);\\n\\njulia> alpha2\\n\\n=\\n\\n-c[2]/(1+r)\\n\\n-c[3]/(1+r)^2;\\n\\njulia> alpha1\\n\\n=\\n\\nc[1]\\n\\n+\\n\\nc[2]/(1+r)\\n\\n+\\n\\nc[3]/(1+r)^2\\n\\n#\\n\\nNPV\\n\\nof\\n\\ncash\\n\\nflow\\n\\n0.18367346938775508\\n\\njulia> alpha1*e1\\n\\n+\\n\\nalpha2*l1\\n\\n+\\n\\nalpha3*l2\\n\\n3-element\\n\\nArray{Float64,1}:\\n\\n1.0\\n\\n2.0\\n\\n-3.0\\n\\n(Later in the course we’ll an automated and simple way to ﬁnd the coeﬃcients in\\n\\nthe expansion of a vector in a basis.)\\n\\n5.3\\n\\nOrthonormal vectors\\n\\nExpansion in an orthonormal basis.\\n\\nLet’s check that the vectors\\n\\na1 =\\n\\n\\uf8ee\\uf8f0\\n\\n0\\n0−1\\n\\n\\uf8f9\\uf8fb ,\\n\\na2 =\\n\\n1√\\n\\n2\\n\\n\\uf8ee\\uf8f0 1\\n\\n1\\n\\n0\\n\\n\\uf8f9\\uf8fb ,\\n\\na3 =\\n\\n1√\\n\\n2\\n\\n\\uf8ee\\uf8f0\\n\\n1−1\\n0\\n\\n\\uf8f9\\uf8fb ,\\n\\nform an orthonormal basis, and check the expansion of x = (1, 2, 3) in this basis,\\n\\n1 x)a1 + · · · + (aT\\nx = (aT\\nn x)an .\\n\\njulia> a1\\n\\n= [0,0,-1];\\n\\na2\\n\\n=\\n\\n[1,1,0]/sqrt(2);\\n\\na3\\n\\n= [1,-1,0]/sqrt(2);\\n\\njulia> norm(a1),\\n\\nnorm(a2),\\n\\nnorm(a3)\\n\\n(1.0,\\n0.9999999999999999,\\n0.9999999999999999)\\njulia> a1'*a2,\\na1'*a3, a2'*a3\\n\\n(0.0,\\n\\n0.0,\\n\\n0.0)\\n\\njulia> x\\n\\n=\\n\\n[1,2,3]\\n\\n3-element\\n\\nArray{Int64,1}:\\n\\n1\\n\\n2\\n\\n3\\n\\njulia> #\\n\\nGet coefficients\\na1'*x;\\n\\nof\\n\\nx\\nin\\northonormal\\nbasis\\na2'*x;\\na3'*x;\\n\\njulia> beta1 =\\n\\nbeta2\\n\\n=\\n\\nbeta3\\n\\n=\\n\\n\\x0c\",\n",
       " '5.4\\n\\nGram–Schmidt algorithm\\n\\n41\\n\\njulia>\\n\\n#\\n\\nExpansion\\n\\nof\\n\\nx\\n\\nin\\n\\nbasis\\n\\njulia>\\n\\nxexp\\n\\n= beta1*a1\\n\\n+\\n\\nbeta2*a2\\n\\n+\\n\\nbeta3*a3\\n\\n3-element Array{Float64,1}:\\n\\n1.0\\n\\n2.0\\n\\n3.0\\n\\n5.4\\n\\nGram–Schmidt algorithm\\n\\nThe following is a Julia implementation of Algorithm 5.1 in VMLS (Gram–Schmidt\\n\\nalgorithm).\\n\\nIt\\n\\ntakes as\\n\\ninput an array [\\n\\na[1],\\n\\na[2], ...,\\n\\na[k]\\n\\n], containing\\n\\nthe k vectors a1 , . . . , ak .\\n] with the orthonormal set of vectors computed by the Gram–\\n\\nIf the vectors are linearly independent,\\n\\nit returns an array\\n\\n[ q[1],\\n\\n..., q[k]\\n\\nSchmidt algorithm.\\n\\nIf\\n\\nthe vectors are linearly dependent and the Gram–Schnidt\\n\\nalgorithm terminates early in iteration i, it returns the array [\\n\\nq[1],\\n\\n..., q[i]\\n\\n]\\n\\nof\\n\\nlength i.\\n\\n1\\n\\n2\\n\\n3\\n\\n4\\n\\n5\\n\\n6\\n\\n7\\n\\n8\\n\\n9\\n\\n10\\n\\n11\\n\\n12\\n\\n13\\n\\n14\\n\\n15\\n\\n16\\n\\nfunction gram_schmidt(a;\\n\\ntol\\n\\n=\\n\\n1e-10)\\n\\nq\\n\\n= []\\n\\nfor i\\n\\n= 1:length(a)\\n\\nqtilde =\\n\\na[i]\\n\\nfor j\\n\\n= 1:i-1\\n\\nqtilde -=\\n\\nend\\n\\n(q[j]\\'*a[i])\\n\\n*\\n\\nq[j]\\n\\nif norm(qtilde)\\n\\n<\\n\\ntol\\n\\nprintln(\"Vectors\\n\\nare\\n\\nlinearly\\n\\ndependent.\")\\n\\nreturn q\\n\\nend\\n\\npush!(q, qtilde/norm(qtilde))\\n\\nend;\\n\\nreturn\\n\\nq\\n\\nend\\n\\nOn line 3, we initialize the output array as the empty array.\\n\\nIn each iteration, we\\n\\nadd the next vector to the array using the push! function (line 13).\\n\\nExample. We apply the function to the example on page 100 of VMLS.\\n\\n\\x0c',\n",
       " \"42\\n\\n5\\n\\nLinear independence\\n\\njulia> a\\n\\n=\\n\\n[ [-1,\\n\\n1,\\n\\n-1,\\n\\n1],\\n\\n[-1,\\n\\n3,\\n\\n-1,\\n\\n3],\\n\\n[1, 3,\\n\\n5,\\n\\n7]\\n\\n]\\n\\n3-element\\n\\nArray{Array{Int64,1},1}:\\n\\n[-1,\\n\\n1, -1, 1]\\n\\n[-1,\\n\\n3, -1, 3]\\n\\n[1, 3,\\n\\n5,\\n\\n7]\\n\\njulia> q\\n\\n=\\n\\ngram_schmidt(a)\\n\\n3-element\\n\\nArray{Any,1}:\\n\\n[-0.5,\\n\\n0.5, -0.5,\\n\\n0.5]\\n\\n[0.5,\\n\\n0.5, 0.5,\\n\\n0.5]\\n\\n[-0.5,\\n\\n-0.5,\\n\\n0.5,\\n\\n0.5]\\n\\njulia> #\\n\\ntest\\n\\northnormality\\n\\njulia> norm(q[1])\\n\\n1.0\\njulia> q[1]'*q[2]\\n\\n0.0\\njulia> q[1]'*q[3]\\n\\n0.0\\n\\njulia> norm(q[2])\\n\\n1.0\\njulia> q[2]'*q[3]\\n\\n0.0\\n\\njulia> norm(q[3])\\n\\n1.0\\n\\nExample of early termination.\\n\\nIf we replace a3 with a linear combination of a1\\nand a2 the set becomes linearly dependent.\\n\\njulia> b\\n\\n=\\n\\n[ a[1],\\n\\na[2],\\n\\n1.3*a[1]\\n\\n+\\n\\n0.5*a[2]\\n\\n]\\n\\n3-element\\n\\nArray{Array{Float64,1},1}:\\n\\n[-1.0,\\n\\n1.0, -1.0,\\n\\n1.0]\\n\\n[-1.0,\\n\\n3.0, -1.0,\\n\\n3.0]\\n\\n[-1.8,\\n\\n2.8, -1.8,\\n\\n2.8]\\n\\njulia> q\\n\\n=\\n\\ngram_schmidt(b)\\n\\nVectors are linearly\\n\\ndependent.\\n\\n2-element\\n\\nArray{Any,1}:\\n\\n[-0.5,\\n\\n0.5, -0.5,\\n\\n0.5]\\n\\n[0.5,\\n\\n0.5, 0.5,\\n\\n0.5]\\n\\nExample\\n\\nof\\n\\nindependence-dimension\\n\\ninequality. We know that\\n\\nany three\\n\\n2-\\n\\nvectors must be dependent. Let’s use the Gram-Schmidt algorithm to verify this\\n\\nfor three speciﬁc vectors.\\n\\n\\x0c\",\n",
       " '5.4\\n\\nGram–Schmidt algorithm\\n\\n43\\n\\njulia>\\n\\nthree_two_vectors\\n\\n=\\n\\n[\\n\\n[1,1],\\n\\n[1,2],\\n\\n[-1,1]\\n\\n]\\n\\njulia>\\n\\nq\\n\\n= gram_schmidt(three_two_vectors)\\n\\nVectors\\n\\nare\\n\\nlinearly\\n\\ndependent.\\n\\n2-element Array{Any,1}:\\n\\n[0.707107, 0.707107]\\n\\n[-0.707107, 0.707107]\\n\\n\\x0c',\n",
       " '44\\n\\n5\\n\\nLinear independence\\n\\n\\x0c',\n",
       " 'Chapter 6\\n\\nMatrices\\n\\n6.1\\n\\nMatrices\\n\\nCreating matrices\\n\\nfrom the\\n\\nentries.\\n\\nMatrices\\n\\nare\\n\\nrepresented in Julia\\n\\nas\\n\\n2-\\n\\ndimensional arrays.\\nThese are\\nconstructed by giving the\\nelements\\nin each row,\\nseparated by space, with the rows separated by semicolons. For example, the 3 × 4\\n\\nmatrix\\n\\nA =\\n\\n\\uf8ee\\uf8f0\\n\\n\\uf8f9\\uf8fb\\n\\n0\\n\\n1.3\\n\\n4.1\\n\\n1\\n\\n4\\n\\n−1\\n\\n−2.3\\n−0.1\\n\\n0\\n\\n0.1\\n\\n0\\n\\n1.7\\n\\nis constructed in Julia as\\n\\njulia>\\n\\nA\\n\\n= [0.0\\n\\n1.0\\n\\n-2.3\\n\\n1.3\\n\\n4.0 -0.1\\n\\n0.1;\\n\\n0.0;\\n\\n4.1 -1.0\\n\\n0.0\\n\\n1.7]\\n\\nArray{Float64,2}:\\n\\n3×4\\n\\n0.0\\n\\n1.3\\n\\n4.1\\n\\n1.0\\n\\n4.0\\n\\n-1.0\\n\\n-2.3\\n\\n-0.1\\n\\n0.0\\n\\n0.1\\n\\n0.0\\n\\n1.7\\n\\n(Here, Array{Float64,2} above the array tells us that the array is 2-dimensional,\\n\\nand its\\n\\nentries are 64-bit ﬂoating-point numbers.)\\n\\nIn this\\n\\nexample, we put\\n\\nthe\\n\\ndiﬀerent rows of the matrix on diﬀerent lines, which makes the code more readable,\\n\\nbut there is no need to do this; we get the same matrix with\\n\\n0.1;\\n\\n1.3\\n\\n4\\n\\n-0.1\\n\\n0;\\n\\n4.1\\n\\n-1\\n\\n0\\n\\n1.7]\\n\\njulia>\\nA\\n= [0 1\\n-2.3\\n3×4 Array{Float64,2}:\\n\\n0.0\\n\\n1.3\\n\\n4.1\\n\\n1.0\\n\\n4.0\\n\\n-1.0\\n\\n-2.3\\n\\n-0.1\\n\\n0.0\\n\\n0.1\\n\\n0.0\\n\\n1.7\\n\\n\\x0c',\n",
       " '46\\n\\n6\\n\\nMatrices\\n\\nThe Julia function size(A) gives\\n\\nthe\\n\\nsize, as a tuple.\\n\\nIt\\n\\ncan also be\\n\\ncalled as\\n\\nsize(A,1) or size(A,2) to get only the number of rows or columns. As an exam-\\n\\nple, we create a function that determines if a matrix is tall.\\n\\njulia> m,\\n\\nn =\\n\\nsize(A)\\n\\n(3, 4)\\n\\njulia> m\\n\\n3\\n\\njulia> n\\n\\n4\\n\\njulia> size(A,1)\\n\\n3\\n\\njulia> size(A,2)\\n\\n4\\n\\njulia> tall(X)\\n\\n=\\n\\nsize(X,1)>size(X,2);\\n\\njulia> tall(A)\\n\\nfalse\\n\\nIn the\\n\\nfunction deﬁnition,\\n\\nthe number of\\n\\nrows and the number of\\n\\ncolumns are\\n\\ncombined using the relational operator <, which gives a Boolean.\\n\\nIndexing entries. We get the i, j entry of a matrix A using A[i,j]. We can also\\n\\nassign a new value to an entry.\\n\\njulia> A[2,3]\\n\\n#\\n\\nGet\\n\\n2,3\\n\\nentry\\n\\nof\\n\\nA\\n\\n-0.1\\n\\njulia> A[1,3]\\n\\n=\\n\\n7.5;\\n\\n#\\n\\nSet\\n\\n1,3\\n\\nentry\\n\\nof\\n\\nA\\n\\nto\\n\\n7.5\\n\\n7.5\\n\\njulia> A\\n3×4\\n\\nArray{Float64,2}:\\n\\n0.0\\n\\n1.3\\n\\n4.1\\n\\n1.0\\n\\n4.0\\n\\n-1.0\\n\\n7.5\\n\\n-0.1\\n\\n0.0\\n\\n0.1\\n\\n0.0\\n\\n1.7\\n\\nSingle\\n\\nindex\\n\\nindexing.\\n\\nJulia allows you to access an entry of a matrix using\\n\\nonly one\\n\\nindex. To use this, you need to know that matrices in Julia are stored\\n\\nin column-major order.\\n\\nThis means\\n\\nthat a matrix can be\\n\\nconsidered as a one-\\n\\ndimensional array, with the ﬁrst column stacked on top of the second, stacked on\\n\\ntop of the third, and so on. For example, the elements of the matrix\\n\\n(cid:20) −1\\n−1\\n\\nZ =\\n\\n(cid:21)\\n\\n0\\n\\n2\\n\\n2\\n\\n3\\n\\nare stored in the order\\n\\n−1, −1, 0, 2, 2, 3.\\n\\n\\x0c',\n",
       " '6.1\\n\\nMatrices\\n\\n47\\n\\nWith single index indexing, Z[5] is the ﬁfth element in this sequence.\\n\\njulia>\\n\\nZ\\n\\n= [ -1\\n\\n0\\n\\n2;\\n\\n-1\\n\\n2\\n\\n-3];\\n\\njulia>\\n\\nZ[5]\\n\\n2\\n\\nThis is very much not\\n\\nstandard mathematical notation, and we would never use\\n\\nthis in VMLS. But it can be handy in some cases when you are using Julia.\\n\\nEquality of matrices.\\n\\nA\\n\\n==\\n\\nB determines whether the matrices A and B are equal.\\n\\nThe expression A\\n\\n.==\\n\\nB creates a matrix whose entries are Boolean, depending\\n\\non whether\\n\\nthe corresponding entries of A and B are the same.\\n\\nThe expression\\n\\nsum(A\\n\\n.==\\n\\nB) gives the number of entries of A and B that are equal.\\n\\njulia>\\n\\nB\\n\\n= copy(A);\\n\\njulia>\\n\\nB[2,2] =\\n\\n0;\\n\\njulia>\\n\\nA\\n\\n==\\n\\nB\\n\\nfalse\\n\\njulia>\\nA\\n.== B\\n3×4 BitArray{2}:\\n\\ntrue\\n\\ntrue\\n\\ntrue\\n\\ntrue\\n\\ntrue\\n\\nfalse\\n\\ntrue\\n\\ntrue\\n\\ntrue\\n\\ntrue\\n\\ntrue\\n\\ntrue\\n\\njulia>\\n\\nsum(A .==\\n\\nB)\\n\\n11\\n\\nRow and column vectors.\\n\\nIn Julia, as in VMLS, n-vectors are the same as n × 1\\n\\nmatrices.\\n\\njulia>\\na\\n= [ -2.1\\n-3\\n1×3 Array{Float64,2}:\\n\\n0\\n\\n]\\n\\n#\\n\\nA\\n\\n3-row\\n\\nvector\\n\\nor\\n\\n1x3\\n\\nmatrix\\n\\n-2.1\\n\\n-3.0\\n\\n0.0\\n\\njulia>\\n\\nb\\n\\n= [ -2.1;\\n\\n-3;\\n\\n0\\n\\n]\\n\\n#\\n\\nA\\n\\n3-vector\\n\\nor\\n\\n3x1\\n\\nmatrix\\n\\n3-element Array{Float64,1}:\\n\\n-2.1\\n\\n-3.0\\n\\n0.0\\n\\nThe output reveals a small subtlety that generally won’t aﬀect you. You can see\\n\\nthat b has\\n\\ntype Array{Float64,1}.\\n\\nThe ﬁnal 1 means\\n\\nthat\\n\\nit\\n\\nis a 1D array;\\n\\nsize(b) gives (3,), whereas you might think it would or should be (3,1). So you\\nin Julia, n × 1 matrices are n-vectors. This is why we say above\\nmight say that\\nthat n-vectors and n × 1 matrices are almost the same in Julia.\\n\\nSlicing and submatrices.\\n\\nUsing colon notation you can extract a submatrix.\\n\\n\\x0c',\n",
       " '48\\n\\n6\\n\\nMatrices\\n\\njulia> A\\n3×4\\n\\nArray{Int64,2}:\\n\\n=\\n\\n[ -1\\n\\n0\\n\\n1\\n\\n0\\n\\n;\\n\\n2\\n\\n-3\\n\\n0\\n\\n1\\n\\n;\\n\\n0\\n\\n4\\n\\n-2\\n\\n1]\\n\\n-1\\n\\n2\\n\\n0\\n\\n0\\n\\n-3\\n\\n4\\n\\n1\\n\\n0\\n\\n-2\\n\\n0\\n\\n1\\n\\n1\\n\\njulia>\\n2×2\\n\\nA[1:2,3:4]\\n\\nArray{Float64,2}:\\n\\n1\\n\\n0\\n\\n0\\n\\n1\\n\\nThis is very similar to the mathematical notation in VMLS, where this submatrix\\n\\nwould be denoted A1:2,3:4 . You can also assign a submatrix using slicing (index\\nrange) notation.\\n\\nA very useful shortcut is the index range : which refers to the whole index range\\n\\nfor that index. This can be used to extract the rows and columns of a matrix.\\n\\njulia> A[:,3]\\n\\n#\\n\\nThird\\n\\ncolumn\\n\\nof\\n\\nA\\n\\n3-element\\n\\nArray{Int64,1}:\\n\\n1\\n\\n0\\n\\n-2\\n\\njulia> A[2,:]\\n\\n#\\n\\nSecond\\n\\nrow\\n\\nof\\n\\nA,\\n\\nreturned\\n\\nas\\n\\ncolumn\\n\\nvector!\\n\\n4-element\\n\\nArray{Int64,1}:\\n\\n2\\n\\n-3\\n\\n0\\n\\n1\\n\\nIn mathematical\\n\\n(VMLS) notation, we say that A[2,:] returns\\n\\nthe transpose of\\n\\nthe second row of A.\\n\\nAs with vectors, Julia’s slicing and selection is not limited to contiguous ranges\\n\\nof\\n\\nindexes. For example, we can reverse the order of the rows of a matrix X using\\n\\njulia> m\\n\\n=\\n\\nsize(X,1)\\n\\njulia> X[m:-1:1,:]\\n\\n#\\n\\nMatrix\\n\\nX\\n\\nwith\\n\\nrow\\n\\norder\\n\\nreversed\\n\\nJulia’s single indexing for matrices can be used with index ranges or sets. For\\nexample if X is an m × n matrix, X[:] is a vector of size mn that consists of the\\ngives a new k × l matrix, with the entries taken in the column-ma jor order from X.\\ncolumns of X stacked on top of each other. The Julia function reshape(X,(k,l))\\n(We must have mn = k l, i.e., the original and reshaped matrix must have the same\\n\\nnumber of entries.) Neither of these is standard mathematical notation, but they\\n\\ncan be useful\\n\\nin Julia.\\n\\n\\x0c',\n",
       " '6.1\\n\\nMatrices\\n\\n49\\n\\njulia>\\nB\\n= [ 1\\n-3\\n;\\n3×2 Array{Int64,2}:\\n\\n2\\n\\n0\\n\\n;\\n\\n1\\n\\n-2]\\n\\n1\\n\\n-3\\n\\n2\\n\\n0\\n\\n1\\n\\n-2\\n\\njulia>\\n\\nB[:]\\n\\n6-element Array{Int64,1}:\\n\\n1\\n\\n2\\n\\n1\\n\\n-3\\n\\n0\\n\\n-2\\n\\njulia>\\nreshape(B,(2,3))\\n2×3 Array{Int64,2}:\\n\\n1\\n\\n1\\n\\n0\\n\\n2\\n\\n-3\\n\\n-2\\n\\njulia>\\n\\nreshape(B,(3,3))\\n\\nERROR:\\n\\nDimensionMismatch(\"new\\n\\ndimensions\\n\\n(3,\\n\\n3)\\n\\nmust\\n\\nbe\\n\\nconsistent\\n\\nwith array size\\n\\n6\")\\n\\nBlock matrices.\\n\\nBlock matrices are\\n\\nconstructed in Julia very much as\\n\\nin the\\n\\nstandard mathematical notation in VMLS. You use ; to stack matrices, and a\\n\\nspace to do (horizontal) concatenation. We apply this to the example on page 109\\n\\nof VMLS.\\n\\njulia>\\n\\nB\\n\\n= [ 0\\n\\n2\\n\\n3\\n\\n];\\n\\n#\\n\\n1x3\\n\\nmatrix\\n\\njulia>\\n\\nC\\n\\n= [ -1\\n\\n];\\n\\n#\\n\\n1x1\\n\\nmatrix\\n\\njulia>\\n\\nD\\n\\n= [ 2\\n\\n2\\n\\n1\\n\\n;\\n\\n1\\n\\n3\\n\\n5];\\n\\n#\\n\\n2x3\\n\\nmatrix\\n\\njulia>\\n\\nE\\n\\n= [4 ;\\n\\n4\\n\\n];\\n\\n#\\n\\n2x1\\n\\nmatrix\\n\\njulia>\\n\\n#\\n\\nconstruct\\n\\n3x4\\n\\nblock\\n\\nmatrix\\n\\njulia>\\n\\nA\\n\\n= [B C\\n\\n;\\n\\nD\\n\\nE]\\n\\n3×4\\n\\nArray{Int64,2}:\\n\\n0\\n\\n2\\n\\n3\\n\\n-1\\n\\n2\\n\\n2\\n\\n1\\n\\n4\\n\\n1\\n\\n3\\n\\n5\\n\\n4\\n\\nColumn and row interpretation of a matrix.\\n\\nAn m × n matrix A can be inter-\\n\\npreted as a collection of n m-vectors (its columns) or a collection of m row vectors\\n\\n(its rows). Julia distinguishes between a matrix (a two-dimensional array) and an\\n\\n\\x0c',\n",
       " '50\\n\\n6\\n\\nMatrices\\n\\narray of vectors. An array (or a tuple) of column vectors can be converted into a\\n\\nmatrix using the horizontal concatenation function hcat.\\n\\njulia> a\\n\\n=\\n\\n[ [1.,\\n\\n2.],\\n\\n[4.,\\n\\n5.],\\n\\n[7.,\\n\\n8.]\\n\\n]\\n\\n#\\n\\narray\\n\\nof\\n\\n2-vectors\\n\\n3-element\\n\\nArray{Array{Float64,1},1}:\\n\\n[1.0,\\n\\n2.0]\\n\\n[4.0,\\n\\n5.0]\\n\\n[7.0,\\n\\n8.0]\\n\\njulia> A\\n2×3\\n\\n=\\n\\nhcat(a...)\\n\\nArray{Float64,2}:\\n\\n1.0\\n\\n2.0\\n\\n4.0\\n\\n5.0\\n\\n7.0\\n\\n8.0\\n\\nThe ... operator in hcat(a...) splits the array a into its elements, i.e., hcat(a...)\\n\\nis\\n\\nthe same as hcat(a[1], a[2],\\n\\na[3]), which concatenates a[1], a[2], a[3]\\n\\nhorizontally.\\n\\nSimilarly, vcat concatenates an array of arrays vertically. This is useful when\\n\\nconstructing a matrix from its row vectors.\\n\\njulia> a\\n\\n=\\n\\n[ [1.\\n\\n2.],\\n\\n[4.\\n\\n5.],\\n\\n[7.\\n\\n8.]\\n\\n]\\n\\n#\\n\\narray of\\n\\n1x2\\n\\nmatrices\\n\\n3-element\\n\\nArray{Array{Float64,2},1}:\\n\\n[1.0\\n\\n2.0]\\n\\n[4.0\\n\\n5.0]\\n\\n[7.0\\n\\n8.0]\\n\\njulia> A\\n3×2\\n\\n=\\n\\nvcat(a...)\\n\\nArray{Float64,2}:\\n\\n1.0\\n\\n4.0\\n\\n7.0\\n\\n2.0\\n\\n5.0\\n\\n8.0\\n\\n6.2\\n\\nZero and identity matrices\\n\\nZero matrices.\\n\\nA zero matrix of size m × n is created using zeros(m,n).\\n\\njulia> zeros(2,2)\\n2×2\\n\\nArray{Float64,2}:\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\nIdentity matrices.\\n\\nIdentity matrices in Julia can be created many ways,\\n\\nfor ex-\\n\\nample by starting with a zero matrix and then setting the diagonal entries to one.\\n\\n\\x0c',\n",
       " '6.2\\n\\nZero and identity matrices\\n\\n51\\n\\nThe LinearAlgebra package also contains functions for creating a special\\n\\nidentity\\n\\nmatrix ob ject I, which has some nice features. You can use 1.0*Matrix(I,n,n) to\\ncreate an n × n identity matrix.\\n\\n(Multiplying by 1.0 converts the matrix into one\\n\\nwith numerical entries; otherwise it has Boolean entries.) This expression is pretty\\n\\nunwieldy, so we can deﬁne a function eye(n) to generate an identity matrix. This\\n\\nfunction is in the VMLS package, so you can use it once the package is installed.\\n\\n(The\\n\\nname eye to denote the identity matrix I traces back to the MATLAB language.)\\n\\njulia>\\n\\neye(n) =\\n\\n1.0*Matrix(I,n,n)\\n\\njulia>\\neye(4)\\n4×4 Array{Float64,2}:\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n1.0\\n\\nJulia’s identity matrix I has some useful properties. For example, when it can\\n\\ndeduce\\n\\nits dimensions, you don’t have\\n\\nto specify it.\\n\\n(This\\n\\nis\\n\\nthe\\n\\nsame as with\\n\\ncommon mathematical notation; see VMLS page 113.)\\n\\njulia>\\nA\\n= [ 1\\n-1\\n2;\\n2×3 Array{Int64,2}:\\n\\n0\\n\\n3\\n\\n-1]\\n\\n1\\n\\n0\\n\\n-1\\n\\n3\\n\\n2\\n\\n-1\\n\\njulia>\\n[A I]\\n2×5 Array{Int64,2}:\\n\\n1\\n\\n0\\n\\n-1\\n\\n3\\n\\n2\\n\\n-1\\n\\n1\\n\\n0\\n\\n0\\n\\n1\\n\\njulia>\\n[A ;\\nI]\\n5×3 Array{Int64,2}:\\n\\n1\\n\\n0\\n\\n1\\n\\n0\\n\\n0\\n\\n-1\\n\\n3\\n\\n0\\n\\n1\\n\\n0\\n\\n2\\n\\n-1\\n\\n0\\n\\n0\\n\\n1\\n\\njulia>\\nB\\n= [ 1\\n2\\n;\\n2×2 Array{Int64,2}:\\n\\n3\\n\\n4\\n\\n]\\n\\n1\\n\\n3\\n\\n2\\n\\n4\\n\\njulia>\\nB\\n+ I\\n2×2 Array{Int64,2}:\\n\\n2\\n\\n3\\n\\n2\\n\\n5\\n\\n\\x0c',\n",
       " '52\\n\\n6\\n\\nMatrices\\n\\nOnes matrix.\\n\\nIn VMLS we do not have notation for a matrix with all entries one.\\n\\nIn Julia, such a matrix is given by ones(m,n).\\n\\nDiagonal matrices.\\n\\nIn standard mathematical notation, diag(1, 2, 3) is a diagonal\\n3 × 3 matrix with diagonal entries 1, 2, 3.\\n\\nIn Julia such a matrix is created using\\n\\nthe\\n\\nfunction diagm, provided in the LinearAlgebra package.\\n\\nTo construct\\n\\ndiagonal matrix with diagonal\\n\\nentries\\n\\nin the vector s, you use diagm(0\\n\\n=>\\n\\nthe\\n\\ns).\\n\\nThis is fairly unwieldy, so the VMLS package deﬁnes a function diagonal(s).\\n\\n(Note\\n\\nthat you have to pass the diagonal entries as a vector.)\\n\\njulia> diagonal(x)\\n\\n=\\n\\ndiagm(0\\n\\n=>\\n\\nx)\\n\\ndiagonal\\n\\n(generic\\n\\nfunction\\n\\nwith\\n\\n1\\n\\nmethod)\\n\\njulia> diagonal([1,2,3])\\n3×3\\n\\nArray{Int64,2}:\\n\\n1\\n\\n0\\n\\n0\\n\\n0\\n\\n2\\n\\n0\\n\\n0\\n\\n0\\n\\n3\\n\\nA closely related Julia function diag(X) does the opposite:\\n\\nIt takes the diagonal\\n\\nentries of the (possibly not square) matrix X and puts them into a vector.\\n\\njulia> H\\n2×4\\n\\nArray{Int64,2}:\\n\\n=\\n\\n[0\\n\\n1\\n\\n-2\\n\\n1;\\n\\n2\\n\\n-1\\n\\n3\\n\\n0]\\n\\n0\\n\\n2\\n\\n1\\n\\n-1\\n\\n-2\\n\\n3\\n\\n1\\n\\n0\\n\\njulia> diag(H)\\n\\n2-element\\n\\nArray{Int64,1}:\\n\\n0\\n\\n-1\\n\\nRandom matrices.\\n\\nA random m×n matrix with entries between 0 and 1 is created\\n\\nusing rand(m,n). For entries that have a normal distribution, randn(m,n).\\n\\njulia> rand(2,3)\\n2×3\\n\\nArray{Float64,2}:\\n\\n0.365832\\n\\n0.381598\\n\\n0.321444\\n\\n0.0317522\\n\\n0.434451\\n\\n0.95419\\n\\njulia> randn(3,2)\\n3×2\\n\\nArray{Float64,2}:\\n\\n0.541546\\n\\n1.65458\\n\\n-0.684011\\n\\n-2.12776\\n\\n0.0443909\\n\\n-1.81297\\n\\n\\x0c',\n",
       " '6.2\\n\\nZero and identity matrices\\n\\n53\\n\\nSparse matrices.\\n\\nFunctions\\n\\nfor\\n\\ncreating and manipulating sparse matrices are\\n\\ncontained in the SparseArrays package, which must be\\n\\ninstalled;\\n\\nsee page\\n\\nix.\\n\\nSparse matrices are stored in a special format that exploits the property that most\\n\\nof the elements are zero. The sparse function creates a sparse matrix from three\\n\\narrays\\n\\nthat\\n\\nspecify the\\n\\nrow indexes,\\n\\ncolumn indexes, and values of\\n\\nthe nonzero\\n\\nelements. The following code creates a sparse matrix\\n\\nA =\\n\\n\\uf8ee\\uf8ef\\uf8ef\\uf8f0 −1.11\\n\\n0\\n−0.10\\n\\n1.17\\n\\n0\\n\\n0\\n\\n0.15\\n\\n0\\n−0.30\\n\\n0\\n\\n0\\n\\n0\\n\\n0\\n\\n0\\n\\n0\\n\\n0\\n\\n0\\n\\n0\\n\\n0.13\\n\\n0\\n\\n\\uf8f9\\uf8fa\\uf8fa\\uf8fb .\\n\\njulia>\\n\\nI\\n\\n= [ 1,\\n\\n2,\\n\\n2,\\n\\n1,\\n\\n3,\\n\\n4\\n\\n]\\n\\n#\\n\\nrow\\n\\nindexes\\n\\nof\\n\\nnonzeros\\n\\njulia>\\n\\nJ\\n\\n= [ 1,\\n\\n1,\\n\\n2,\\n\\n3,\\n\\n3,\\n\\n4\\n\\n]\\n\\n#\\n\\ncolumn\\n\\nindexes\\n\\njulia>\\n\\nV\\n\\n= [ -1.11,\\n\\n0.15,\\n\\n-0.10,\\n\\n1.17,\\n\\n-0.30,\\n\\n0.13\\n\\n]\\n\\n#\\n\\nvalues\\n\\njulia>\\nA\\n= sparse(I,\\nJ,\\nV,\\n4,\\n5)\\n4×5 SparseMatrixCSC{Float64,Int64}\\n\\nwith\\n\\n6\\n\\nstored\\n\\nentries:\\n\\n[1, 1]\\n\\n=\\n\\n-1.11\\n\\n[2, 1]\\n\\n=\\n\\n0.15\\n\\n[2, 2]\\n\\n=\\n\\n-0.1\\n\\n[1, 3]\\n\\n=\\n\\n1.17\\n\\n[3, 3]\\n\\n=\\n\\n-0.3\\n\\n[4, 4]\\n\\n=\\n\\n0.13\\n\\njulia>\\n\\nnnz(A)\\n\\n6\\n\\nSparse matrices can be converted to regular non-sparse matrices using the Array\\n\\nfunction. Applying sparse to a full matrix gives the equivalent sparse matrix.\\n\\njulia>\\n\\nA\\n\\n= sparse([1,\\n\\n3,\\n\\n2,\\n\\n1],\\n\\n[1,\\n\\n1,\\n\\n2,\\n\\n3],\\n\\n[1.0, 2.0,\\n3×3\\n\\n3.0,\\n\\n4.0],\\n\\n3,\\n\\n3)\\n\\njulia>\\n\\nSparseMatrixCSC{Float64,Int64}\\n\\nwith\\n\\n4\\n\\nstored\\n\\nentries:\\n\\n[1, 1]\\n\\n=\\n\\n1.0\\n\\n[3, 1]\\n\\n=\\n\\n2.0\\n\\n[2, 2]\\n\\n=\\n\\n3.0\\n\\n[1, 3]\\n\\n=\\n\\n4.0\\n\\njulia>\\nB =\\nArray(A)\\n3×3 Array{Float64,2}:\\n\\n1.0\\n\\n0.0\\n\\n4.0\\n\\n0.0\\n\\n3.0\\n\\n0.0\\n\\n2.0\\n\\n0.0\\n\\n0.0\\n\\njulia>\\n\\nB[1,3]\\n\\n=\\n\\n0.0;\\n\\njulia>\\n\\nsparse(B)\\n\\n\\x0c',\n",
       " \"54\\n\\n6\\n\\nMatrices\\n\\n3×3\\n\\nSparseMatrixCSC{Float64,Int64}\\n\\nwith\\n\\n3\\n\\nstored\\n\\nentries:\\n\\n[1, 1]\\n\\n=\\n\\n1.0\\n\\n[3, 1]\\n\\n=\\n\\n2.0\\n\\n[2, 2]\\n\\n=\\n\\n3.0\\n\\nA sparse m × n zero matrix is created with spzeros(m,n). To create a sparse\\nn × n identity matrix in Julia, use sparse(1.0I,n,n). This is not a particularly\\nnatural syntax, so we deﬁne a function speye(n) in the VMLS package. The VMLS\\npackage also includes the function speye(n) which creates a sparse n × n identity\\nmatrix, as well as spdiagonal(a), which creates a sparse diagonal matrix with the\\n\\nentries of the vector a on its diagonal.\\n\\nA useful\\n\\nfunction for creating a random sparse matrix is sprand(m,n,d) (with\\n\\nentries between 0 and 1) and sprandn(m,n,d) (with entries\\n\\nthat\\n\\nrange over all\\n\\nnumbers). The ﬁrst two arguments give the dimensions of the matrix; the last one,\\n\\nd, gives the density of nonzero entries. The nonzero entries are chosen randomly,\\nwith about mnd of them nonzero. The following code creates a random 10000 ×\\n10000 sparse matrix, with a density 10−7 . This means that we’d expect there to\\n\\nbe around 10 nonzero entries.\\n\\n(So this is a very sparse matrix!)\\n\\njulia> A\\n=\\n10000×10000\\n\\nsprand(10000,10000,10^-7)\\n\\nSparseMatrixCSC{Float64,Int64}\\n\\nwith\\n\\n10\\n\\nstored\\n\\nentries:\\n\\n[6435\\n\\n,\\n\\n513]\\n\\n=\\n\\n0.912633\\n\\n[3274\\n\\n,\\n\\n1518]\\n\\n=\\n\\n0.369523\\n\\n[8264\\n\\n,\\n\\n2169]\\n\\n=\\n\\n0.875005\\n\\n[8029\\n\\n,\\n\\n3513]\\n\\n=\\n\\n0.0670725\\n\\n[7427\\n\\n,\\n\\n4122]\\n\\n=\\n\\n0.0376876\\n\\n[6329\\n\\n,\\n\\n5078]\\n\\n=\\n\\n0.970446\\n\\n[3882\\n\\n,\\n\\n5213]\\n\\n=\\n\\n0.0964994\\n\\n[102\\n\\n,\\n\\n6572]\\n\\n=\\n\\n0.697033\\n\\n[730\\n\\n,\\n\\n7936]\\n\\n=\\n\\n0.489414\\n\\n[7006\\n\\n,\\n\\n8422]\\n\\n=\\n\\n0.909351\\n\\n6.3\\n\\nTranspose, addition, and norm\\n\\nTranspose.\\n\\nIn VMLS we denote the transpose of an m × n matrix A as AT .\\nJulia, the transpose of A is given by A'.\\n\\nIn\\n\\njulia>\\n2×4\\n\\nH\\n\\n= [0\\n\\n1\\n\\n-2\\n\\n1;\\n\\n2\\n\\n-1\\n\\n3\\n\\n0]\\n\\nArray{Int64,2}:\\n\\n0\\n\\n1\\n\\n-2\\n\\n1\\n\\n2\\n\\n-1\\n\\n3\\n\\n0\\n\\n\\x0c\",\n",
       " \"6.3\\n\\nTranspose, addition, and norm\\n\\n55\\n\\nH'\\n\\njulia>\\n4×2 Array{Int64,2}:\\n\\n0\\n\\n1\\n\\n-2\\n\\n1\\n\\n2\\n\\n-1\\n\\n3\\n\\n0\\n\\nAddition, subtraction, and scalar multiplication.\\n\\nIn Julia, addition and subtrac-\\n\\ntion of matrices, and scalar-matrix multiplication, both follow standard mathemat-\\n\\nical notation.\\n\\njulia>\\nU\\n= [ 0\\n4;\\n7\\n3×2 Array{Int64,2}:\\n\\n0;\\n\\n3\\n\\n1]\\n\\n0\\n\\n7\\n\\n3\\n\\n4\\n\\n0\\n\\n1\\n\\njulia>\\nV\\n= [ 1\\n2;\\n2\\n3×2 Array{Int64,2}:\\n\\n3;\\n\\n0\\n\\n4]\\n\\n1\\n\\n2\\n\\n0\\n\\n2\\n\\n3\\n\\n4\\n\\njulia>\\nU+V\\n3×2 Array{Int64,2}:\\n\\n1\\n\\n9\\n\\n3\\n\\n6\\n\\n3\\n\\n5\\n\\njulia>\\n2.2*U\\n3×2 Array{Float64,2}:\\n\\n0.0\\n\\n15.4\\n\\n6.6\\n\\n8.8\\n\\n0.0\\n\\n2.2\\n\\n(We can also multiply a matrix on the right by a scalar.)\\n\\nJulia supports some operations that are not standard mathematical ones. For\\n\\nexample,\\n\\nin Julia you can add or subtract a constant from a matrix, which carries\\n\\nout the operation on each entry.\\n\\nElementwise operations.\\n\\nThe syntax for elementwise vector operations described\\n\\non page 10 carries over naturally to matrices. We add a period before a binary\\n\\noperator\\n\\nto change the interpretation to elementwise.\\n\\nFor example,\\n\\nif A and B\\n\\nare matrices of the same size, then C\\n\\n=\\n\\nA .*\\n\\nB creates a matrix of the same size\\n\\nwith elements Cij = Aij Bij . We can add a period after a function name to tell\\nJulia that the function should be applied elementwise. Thus,\\nif X is a matrix, then\\n\\nY = exp.(X) creates a matrix of the same size, with elements Yij = exp(Xij ).\\n\\n\\x0c\",\n",
       " '56\\n\\n6\\n\\nMatrices\\n\\nMatrix norm.\\n\\nIn VMLS we use (cid:107)A(cid:107) to denote the norm of an m × n matrix,\\n\\n\\uf8eb\\uf8ed m(cid:88)\\n\\nn(cid:88)\\n\\n(cid:107)A(cid:107) =\\n\\n\\uf8f6\\uf8f81/2\\n\\n.\\n\\nA2\\n\\nij\\n\\ni=1\\n\\nj=1\\n\\nIn standard mathematical notation, this is more often written as (cid:107)A(cid:107)F , where F\\nIn standard mathematical notation, (cid:107)A(cid:107) usually\\nstands for the name Frobenius.\\n\\nrefers to another norm of a matrix, that is beyond the scope of the topics in VMLS.\\n\\nIn Julia, norm(A) gives the norm used in VMLS.\\n\\njulia> A\\n\\n=\\n\\n[2\\n\\n3\\n\\n-1;\\n\\n0\\n\\n-1\\n\\n4]\\n\\njulia> norm(A)\\n\\n5.5677643628300215\\n\\njulia> norm(A[:])\\n\\n5.5677643628300215\\n\\nTriangle inequality.\\n\\nLet’s check that the triangle inequality (cid:107)A + B (cid:107) ≤ (cid:107)A(cid:107) + (cid:107)B (cid:107)\\n\\nholds,\\n\\nfor two speciﬁc matrices.\\n\\njulia> A\\n\\n=\\n\\n[-1\\n\\n0;\\n\\n2\\n\\n2];\\n\\nB=\\n\\n[3\\n\\n1;\\n\\n-3\\n\\n2];\\n\\njulia> norm(A\\n\\n+\\n\\nB),\\n\\nnorm(A)\\n\\n+\\n\\nnorm(B)\\n\\n(4.69041575982343,\\n\\n7.795831523312719)\\n\\n6.4\\n\\nMatrix-vector multiplication\\n\\nIn Julia, matrix-vector multiplication has the natural syntax y=A*x.\\n\\njulia> A\\n2×3\\n\\nArray{Int64,2}:\\n\\n=\\n\\n[0\\n\\n2\\n\\n-1;\\n\\n-2\\n\\n1\\n\\n1]\\n\\n0\\n\\n-2\\n\\n2\\n\\n1\\n\\n-1\\n\\n1\\n\\njulia>\\n\\nx = [2,\\n\\n1,\\n\\n-1]\\n\\n3-element\\n\\nArray{Int64,1}:\\n\\n2\\n\\n1\\n\\n-1\\n\\njulia> A*x\\n\\n2-element\\n\\nArray{Int64,1}:\\n\\n3\\n\\n-4\\n\\n\\x0c',\n",
       " '6.4\\n\\nMatrix-vector multiplication\\n\\n57\\n\\nDiﬀerence matrix.\\n\\nAn (n − 1) × n diﬀerence matrix (equation (6.5) of VMLS)\\n\\ncan be constructed in several ways. A simple one is the following.\\n\\njulia>\\n\\ndifference_matrix(n)\\n\\n=\\n\\n[-eye(n-1)\\n\\nzeros(n-1)]\\n\\n+\\n\\n[zeros(n-1),\\n\\neye(n-1)];\\n\\njulia> D\\n3×4\\n\\n=\\n\\ndifference_matrix(4)\\n\\nArray{Float64,2}:\\n\\n-1.0\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n-1.0\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n-1.0\\n\\n1.0\\n\\njulia> D*[-1,0,2,1]\\n\\n3-element\\n\\nArray{Float64,1}:\\n\\n1.0\\n\\n2.0\\n\\n-1.0\\n\\nSince a diﬀerence matrix contains many zeros,\\n\\nthis\\n\\nis a good opportunity to use\\n\\nsparse matrices.\\n\\njulia>\\n\\ndifference_matrix(n)\\n\\n=\\n\\n[-speye(n-1)\\n\\nspzeros(n-1)]\\n\\n+\\n\\n[spzeros(n-1)\\n\\nspeye(n-1)];\\n\\njulia> D\\n3×4\\n\\n=\\n\\ndifference_matrix(4)\\n\\nSparseMatrixCSC{Float64,Int64}\\n\\nwith\\n\\n6\\n\\nstored\\n\\nentries:\\n\\n[1,\\n\\n1]\\n\\n=\\n\\n-1.0\\n\\n[1,\\n\\n2]\\n\\n=\\n\\n1.0\\n\\n[2,\\n\\n2]\\n\\n=\\n\\n-1.0\\n\\n[2,\\n\\n3]\\n\\n=\\n\\n1.0\\n\\n[3,\\n\\n3]\\n\\n=\\n\\n-1.0\\n\\n[3,\\n\\n4]\\n\\n=\\n\\n1.0\\n\\njulia> D*[-1,0,2,1]\\n\\n3-element\\n\\nArray{Float64,1}:\\n\\n1.0\\n\\n2.0\\n\\n-1.0\\n\\nRunning sum matrix.\\n\\nThe running sum matrix (equation (6.6)\\n\\nin VMLS)\\n\\nis a\\n\\nlower triangular matrix, with elements on and below the diagonal equal to one.\\n\\njulia>\\n\\nfunction\\n\\nrunning_sum(n)\\n\\n#\\n\\nn\\n\\nx\\n\\nn\\n\\nrunning\\n\\nsum\\n\\nmatrix\\n\\nS\\n\\n= zeros(n,n)\\n\\nfor i=1:n\\n\\nfor j=1:i\\n\\nS[i,j] =\\n\\n1\\n\\n\\x0c',\n",
       " '58\\n\\n6\\n\\nMatrices\\n\\nend\\n\\nend\\n\\nreturn\\n\\nS\\n\\nend\\n\\nrunning_sum (generic\\n\\nfunction\\n\\nwith\\n\\n1\\n\\nmethod)\\n\\njulia> running_sum(4)\\n4×4\\n\\nArray{Float64,2}:\\n\\n1.0\\n\\n1.0\\n\\n1.0\\n\\n1.0\\n\\n0.0\\n\\n1.0\\n\\n1.0\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n1.0\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n1.0\\n\\njulia> running_sum(4)*[-1,1,2,0]\\n\\n4-element\\n\\nArray{Float64,1}:\\n\\n-1.0\\n\\n0.0\\n\\n2.0\\n\\n2.0\\n\\nAn alternative construction is tril(ones(n,n)). This uses the function tril,\\n\\nwhich sets the elements of a matrix above the diagonal to zero.\\n\\nVandermonde matrix.\\n\\ni\\n\\nAn m × n Vandermonde matrix (equation (6.7) in VMLS)\\n\\ntj−1\\nhas\\nentries\\nfor\\ni = 1, . . . , m and j = 1, . . . , n. We deﬁne a function that\\ntakes an m-vector with elements t1 , . . . , tm and returns the corresponding m × n\\nVandermonde matrix.\\n\\njulia> function\\n\\nvandermonde(t,n)\\n\\nm\\n\\nV\\n\\n= length(t)\\n\\n= zeros(m,n)\\n\\nfor i=1:m\\n\\nfor j=1:n\\n\\nV[i,j]\\n\\n=\\n\\nt[i]^(j-1)\\n\\nend\\n\\nend\\n\\nreturn\\n\\nV\\n\\nend\\n\\nvandermonde (generic\\n\\nfunction\\n\\nwith\\n\\n1\\n\\nmethod)\\n\\njulia> vandermonde([-1,0,0.5,1],5)\\n4×5\\n\\nArray{Float64,2}:\\n\\n1.0\\n\\n1.0\\n\\n1.0\\n\\n1.0\\n\\n-1.0\\n\\n0.0\\n\\n0.5\\n\\n1.0\\n\\n1.0\\n\\n0.0\\n\\n0.25\\n\\n1.0\\n\\n-1.0\\n\\n0.0\\n\\n1.0\\n\\n0.0\\n\\n0.125\\n\\n0.0625\\n\\n1.0\\n\\n1.0\\n\\n\\x0c',\n",
       " '6.5\\n\\nComplexity\\n\\n59\\n\\nAn alternative shorter deﬁnition uses Julia’s hcat function.\\n\\njulia>\\n\\nvandermonde(t,n)\\n\\n=\\n\\nhcat(\\n\\n[t.^i\\n\\nfor\\n\\ni\\n\\n=\\n\\n0:n-1]...\\n\\n)\\n\\nvandermonde\\n\\n(generic\\n\\nfunction\\n\\nwith\\n\\n1\\n\\nmethod)\\n\\njulia>\\nvandermonde([-1,0,0.5,1],5)\\n4×5 Array{Float64,2}:\\n\\n1.0\\n\\n-1.0\\n\\n1.0\\n\\n-1.0\\n\\n1.0\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n1.0\\n\\n0.5\\n\\n0.25\\n\\n0.125\\n\\n0.0625\\n\\n1.0\\n\\n1.0\\n\\n1.0\\n\\n1.0\\n\\n1.0\\n\\n6.5\\n\\nComplexity\\n\\nComplexity of matrix-vector multiplication.\\n\\nThe complexity of multiplying an\\nm × n matrix by an n-vector is 2mn ﬂops. This grows linearly with both m and n.\\n\\nLet’s check this.\\n\\njulia>\\n\\nA\\n\\n= rand(1000,10000);\\n\\nx\\n\\n=\\n\\nrand(10000);\\n\\njulia>\\n\\n@time y\\n\\n=\\n\\nA*x;\\n\\n0.022960\\n\\nseconds\\n\\n(2.01\\n\\nk\\n\\nallocations:\\n\\n127.499\\n\\nKiB)\\n\\njulia>\\n\\n@time y\\n\\n=\\n\\nA*x;\\n\\n0.006321\\n\\nseconds\\n\\n(5\\n\\nallocations:\\n\\n8.094\\n\\nKiB)\\n\\njulia>\\n\\nA\\n\\n= rand(5000,20000);\\n\\nx\\n\\n=\\n\\nrand(20000);\\n\\njulia>\\n\\n@time y\\n\\n=\\n\\nA*x;\\n\\n0.084710\\n\\nseconds\\n\\n(6\\n\\nallocations:\\n\\n39.297\\n\\nKiB)\\n\\njulia>\\n\\n@time y\\n\\n=\\n\\nA*x;\\n\\n0.047996\\n\\nseconds\\n\\n(6\\n\\nallocations:\\n\\n39.297\\n\\nKiB)\\n\\nIn the second matrix-vector multiply, m increases by a factor of 5 and n increases\\n\\nby a factor of 2, so the complexity predicts that the computation time should be\\n\\n(approximately)\\n\\nincreased by a factor of 10. As we can see,\\n\\nit\\n\\nis\\n\\nincreased by a\\n\\nfactor around 7.4.\\n\\nThe increase in eﬃciency obtained by sparse matrix computations is seen from\\n\\nmatrix-vector multiplications with the diﬀerence matrix.\\n\\njulia>\\n\\nn\\n\\n= 10^4;\\n\\njulia>\\n\\nD\\n\\n= [-eye(n-1)\\n\\nzeros(n-1)]\\n\\n+\\n\\n[zeros(n-1)\\n\\neye(n-1)];\\n\\njulia>\\n\\nx\\n\\n= randn(n);\\n\\njulia>\\n\\n@time y=D*x;\\n\\n0.051516 seconds\\n\\n(6\\n\\nallocations:\\n\\n78.359\\n\\nKiB)\\n\\njulia>\\n\\nDs =\\n\\n[-speye(n-1)\\n\\nspzeros(n-1)]\\n\\n+\\n\\n[spzeros(n-1)\\n\\nspeye(n-1)];\\n\\njulia>\\n\\n@time y=Ds*x;\\n\\n\\x0c',\n",
       " '60\\n\\n6\\n\\nMatrices\\n\\n0.000177 seconds\\n\\n(6\\n\\nallocations:\\n\\n78.359\\n\\nKiB)\\n\\n\\x0c',\n",
       " 'Chapter 7\\n\\nMatrix examples\\n\\n7.1\\n\\nGeometric transformations\\n\\nLet’s create a rotation matrix, and use it to rotate a set of points π/3 radians (60◦ ).\\n\\nThe result is in Figure 7.1.\\n\\njulia>\\n\\nRot(theta)\\n\\n=\\n\\n[cos(theta)\\n\\n-sin(theta);\\n\\nsin(theta)\\n\\ncos(theta)];\\n\\njulia>\\nR\\n= Rot(pi/3)\\n2×2 Array{Float64,2}:\\n\\n0.5\\n\\n-0.866025\\n\\n0.866025\\n\\n0.5\\n\\njulia>\\n\\n#\\n\\nCreate\\n\\njulia>\\n\\npoints =\\n\\na\\n\\n[\\n\\n[1,.5]\\n\\n];\\n\\nlist\\n\\nof\\n\\n2-D\\n\\npoints\\n\\n[1,0],\\n\\n[1.5,0],\\n\\n[2,0],\\n\\n[1,0.25],\\n\\n[1.5,\\n\\n0.25],\\n\\njulia> #\\n\\nNow rotate\\n\\nthem.\\n\\njulia> rpoints\\n\\n=\\n\\n[\\n\\nR*p\\n\\nfor\\n\\np\\n\\nin\\n\\npoints\\n\\n];\\n\\njulia> #\\n\\nShow\\n\\nthe\\n\\ntwo\\n\\nsets\\n\\nof\\n\\npoints.\\n\\njulia> using Plots\\n\\njulia> scatter([c[1]\\n\\nfor\\n\\nc\\n\\nin\\n\\npoints],\\n\\n[c[2]\\n\\nfor c\\n\\nin\\n\\npoints])\\n\\njulia> scatter!([c[1]\\n\\nfor\\n\\nc\\n\\nin\\n\\nrpoints],\\n\\n[c[2]\\n\\nfor\\n\\nc\\n\\nin\\n\\nrpoints])\\n\\njulia> plot!(lims\\n\\n=\\n\\n(-0.1,\\n\\n2.1),\\n\\nsize\\n\\n=\\n\\n(500,500),\\n\\nlegend\\n\\n=\\n\\nfalse)\\n\\n7.2\\n\\nSelectors\\n\\nReverser matrix.\\n\\nThe\\n\\nreverser matrix can be\\n\\ncreated from an identity matrix\\n\\nby reversing the order of\\n\\nits rows. The Julia command reverse can be used for\\n\\nthis purpose.\\n\\n(reverse(A,dims=1) reverses\\n\\nthe order of\\n\\nthe\\n\\nrows of a matrix;\\n\\n\\x0c',\n",
       " '62\\n\\n7\\n\\nMatrix examples\\n\\nFigure 7.1 Counterclockwise rotation by 60 degrees applied to six points.\\n\\nflipdim(A,dims=2) reverses the order of the columns.) Multiplying a vector with\\n\\na reverser matrix is the same as reversing the order of\\n\\nits entries directly.\\n\\njulia> reverser(n)\\n\\n=\\n\\nreverse(eye(n),dims=1)\\n\\nreverser\\n\\n(generic\\n\\nfunction\\n\\nwith\\n\\n1\\n\\nmethod)\\n\\njulia> A\\n5×5\\n\\n=\\n\\nreverser(5)\\n\\nArray{Float64,2}:\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\njulia> x\\n\\n=\\n\\n[1.,\\n\\n2.,\\n\\n3.,\\n\\n4.,\\n\\n5.];\\n\\njulia> A*x\\n\\n#\\n\\nReverse\\n\\nx\\n\\nby\\n\\nmultiplying\\n\\nwith\\n\\nreverser\\n\\nmatrix.\\n\\n5-element\\n\\nArray{Float64,1}:\\n\\n5.0\\n\\n4.0\\n\\n3.0\\n\\n2.0\\n\\n1.0\\n\\njulia> reverse(x)\\n\\n#\\n\\nReverse\\n\\nx\\n\\ndirectly.\\n\\n5-element\\n\\nArray{Float64,1}:\\n\\n5.0\\n\\n0.00.51.01.52.00.00.51.01.52.0\\x0c',\n",
       " '7.3\\n\\nIncidence matrix\\n\\n63\\n\\n4.0\\n\\n3.0\\n\\n2.0\\n\\n1.0\\n\\nPermutation matrix.\\n\\nLet’s create a permutation matrix and use it\\n\\nto permute\\n\\nthe entries of a vector.\\n\\nIn Julia,\\n\\nthere is no reason to create a matrix to carry\\n\\nout\\n\\nthe permutation,\\n\\nsince we can do the same thing directly by passing in the\\n\\npermuted indexes to the vector.\\n\\njulia>\\nA\\n= [0 0\\n1;\\n3×3 Array{Int64,2}:\\n\\n1\\n\\n0\\n\\n0;\\n\\n0\\n\\n1\\n\\n0]\\n\\n0\\n\\n1\\n\\n0\\n\\n0\\n\\n0\\n\\n1\\n\\njulia>\\n\\n1\\n\\n0\\n\\n0\\n\\nx\\n\\n= [0.2,\\n\\n-1.7,\\n\\n2.4]\\n\\n3-element Array{Float64,1}:\\n\\n0.2\\n\\n-1.7\\n\\n2.4\\n\\njulia>\\n\\nA*x\\n\\n# Permutes\\n\\nentries\\n\\nof\\n\\nx\\n\\nto\\n\\n[x[3],x[1],x[2]]\\n\\n3-element Array{Float64,1}:\\n\\n2.4\\n\\n0.2\\n\\n-1.7\\n\\njulia>\\n\\nx[[3,1,2]]\\n\\n#\\n\\nSame\\n\\nthing\\n\\nusing\\n\\npermuted\\n\\nindices\\n\\n3-element Array{Float64,1}:\\n\\n2.4\\n\\n0.2\\n\\n-1.7\\n\\n7.3\\n\\nIncidence matrix\\n\\nIncidence matrix of a graph. We\\n\\ncreate\\n\\nthe\\n\\nincidence matrix of\\n\\nthe network\\n\\nshown in Figure 7.3 in VMLS.\\n\\njulia>\\nA\\n= [ -1\\n-1\\n4×5 Array{Int64,2}:\\n\\n-1\\n\\n-1\\n\\n0\\n\\n1\\n\\n0\\n\\n0\\n\\n1\\n\\n0;\\n\\n1\\n\\n0\\n\\n-1\\n\\n0\\n\\n0\\n\\n;\\n\\n0\\n\\n0\\n\\n1\\n\\n-1\\n\\n-1\\n\\n;\\n\\n0\\n\\n1\\n\\n0\\n\\n0\\n\\n1]\\n\\n\\x0c',\n",
       " \"64\\n\\n7\\n\\nMatrix examples\\n\\n1\\n\\n0\\n\\n-1\\n\\n0\\n\\n0\\n\\n0\\n\\n0\\n\\n1\\n\\n-1\\n\\n-1\\n\\n0\\n\\n1\\n\\n0\\n\\n0\\n\\n1\\n\\njulia>\\n\\nxcirc\\n\\n=\\n\\n[1,\\n\\n-1,\\n\\n1,\\n\\n0,\\n\\n1]\\n\\n#\\n\\nA\\n\\ncirculation\\n\\n5-element\\n\\nArray{Int64,1}:\\n\\n1\\n\\n-1\\n\\n1\\n\\n0\\n\\n1\\n\\njulia>\\n\\nA*xcirc\\n\\n4-element\\n\\nArray{Int64,1}:\\n\\n0\\n\\n0\\n\\n0\\n\\n0\\n\\njulia>\\n\\ns = [1,0,-1,0];\\n\\n# A\\n\\nsource\\n\\nvector\\n\\njulia>\\n\\nx = [0.6,\\n\\n0.3,\\n\\n0.6,\\n\\n-0.1,\\n\\n-0.3];\\n\\n#\\n\\nA\\n\\nflow vector\\n\\njulia>\\n\\nA*x +\\n\\ns\\n\\n#\\n\\nTotal incoming\\n\\nflow\\n\\nat\\n\\neach\\n\\nnode\\n\\n4-element\\n\\nArray{Float64,1}:\\n\\n1.11022e-16\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\nDirichlet energy.\\n\\nOn page 135 of VMLS we compute the Dirichlet energy of two\\n\\npotential vectors associated with the graph of Figure 7.2 in VMLS.\\n\\njulia> A\\n4×5\\n\\n=\\n\\n[ -1\\n\\n-1\\n\\n0\\n\\n1\\n\\n0\\n\\n;\\n\\n1\\n\\n0\\n\\n-1\\n\\n0\\n\\n0\\n\\n;\\n\\n0\\n\\n0\\n\\n1\\n\\n-1 -1;\\n\\n0\\n\\n1\\n\\n0\\n\\n0\\n\\n1\\n\\n]\\n\\nArray{Int64,2}:\\n\\n-1\\n\\n-1\\n\\n0\\n\\n1\\n\\n0\\n\\n1\\n\\n0\\n\\n-1\\n\\n0\\n\\n0\\n\\n0\\n\\n0\\n\\n1\\n\\n-1\\n\\n-1\\n\\n0\\n\\n1\\n\\n0\\n\\n0\\n\\n1\\n\\njulia>\\n\\nvsmooth\\n=\\n[\\n1,\\n2,\\nnorm(A'*vsmooth)^2\\n\\n2,\\n\\n1\\n\\n]\\n\\njulia>\\n\\n#\\n\\nDirichlet\\n\\nenergy\\n\\nof\\n\\nvsmooth\\n\\n2.9999999999999996\\n\\njulia> vrough\\n=\\n[ 1,\\n-1,\\njulia> norm(A'*vrough)^2\\n\\n2,\\n\\n-1\\n\\n]\\n\\n#\\n\\nDirichlet\\n\\nenergy\\n\\nof\\n\\nvrough\\n\\n27.0\\n\\n\\x0c\",\n",
       " '7.4\\n\\nConvolution\\n\\n7.4\\n\\nConvolution\\n\\n65\\n\\nThe Julia package DSP includes a convolution function conv.\\n\\nAfter adding this\\n\\npackage, the command conv(a,b) can be used to compute the convolution of the\\n\\nvectors a and b. Let’s use this to ﬁnd the coeﬃcients of the polynomial\\n\\np(x) = (1 + x)(2 − x + x2 )(1 + x − 2x2 ) = 2 + 3x − 3x2 − x3 + x4 − 2x5 .\\n\\njulia>\\n\\nUsing DSP\\n\\njulia>\\n\\njulia>\\n\\njulia>\\n\\njulia>\\n\\na\\n\\nb\\n\\nc\\n\\nd\\n\\n= [1,1];\\n\\n#\\n\\ncoefficients\\n\\nof\\n\\n1+x\\n\\n= [2,-1,1];\\n\\n= [1,1,-2];\\n\\n#\\n\\n#\\n\\ncoefficients\\n\\ncoefficients\\n\\nof\\n\\nof\\n\\n2-x+x^2\\n\\n1+x-2x^2\\n\\n= conv(conv(a,b),c)\\n\\n#\\n\\ncoefficients\\n\\nof\\n\\nproduct\\n\\n6-element Array{Int64,1}:\\n\\n2\\n\\n3\\n\\n-3\\n\\n-1\\n\\n1\\n\\n-2\\n\\nLet’s write a function that creates a Toeplitz matrix, and check it against the\\n\\nconv function. We will also check that Julia is using the very eﬃcient method for\\n\\ncomputing the convolution.\\n\\nTo construct the Toeplitz matrix T (b) deﬁned in equation (7.3) of VMLS, we\\n((n + m − 1) × n) and then\\n\\nthe correct dimensions\\n\\nﬁrst create a zero matrix of\\n\\nadd the coeﬃcients bi one by one.\\npurpose.\\nThe\\nsingle-index indexes of\\nthe\\nelements\\n. . . , i + (n − 1)(m + n).\\n\\ni + m + n, i + 2(m + n),\\n\\nSingle-index indexing comes in handy for this\\n\\nbi\\n\\nin the matrix T (b) are\\n\\ni,\\n\\njulia>\\n\\nfunction\\n\\ntoeplitz(b,n)\\n\\nm\\n\\nT\\n\\n= length(b)\\n\\n= zeros(n+m-1,n)\\n\\nfor i=1:m\\n\\nT[i :\\n\\nn+m\\n\\n: end]\\n\\n.=\\n\\nb[i]\\n\\nend\\n\\nreturn\\n\\nT\\n\\nend\\n\\njulia> b\\n\\n=\\n\\n[-1,2,3];\\n\\na\\n\\n=\\n\\n[-2,3,-1,1];\\n\\njulia> Tb\\n6×4\\n\\nArray{Float64,2}:\\n\\n= toeplitz(b,\\n\\nlength(a))\\n\\n-1.0\\n\\n2.0\\n\\n3.0\\n\\n0.0\\n\\n-1.0\\n\\n0.0\\n\\n0.0\\n\\n2.0\\n\\n-1.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n\\x0c',\n",
       " '66\\n\\n7\\n\\nMatrix examples\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n3.0\\n\\n0.0\\n\\n0.0\\n\\n2.0\\n\\n3.0\\n\\n0.0\\n\\n-1.0\\n\\n2.0\\n\\n3.0\\n\\njulia>\\n\\nTb*a,\\n\\nconv(b,a)\\n\\n([2.0,\\n\\n-7.0,\\n\\n1.0,\\n\\n6.0,\\n\\n-1.0,\\n\\n3.0],\\n\\n[2,\\n\\n-7,\\n\\n1,\\n\\n6, -1,\\n\\n3])\\n\\njulia>\\n\\nm = 2000;\\n\\nn\\n\\n=\\n\\n2000;\\n\\njulia>\\n\\nb = randn(n);\\n\\na=randn(m);\\n\\njulia>\\n\\n@time\\n\\nctoep\\n\\n=\\n\\ntoeplitz(b,n)*a;\\n\\n0.124865 seconds\\n\\n(8.01\\n\\nk\\n\\nallocations:\\n\\n122.696\\n\\nMiB,\\n\\n5.07%\\n\\ngc\\n\\ntime)\\n\\njulia>\\n\\n@time\\n\\ncconv\\n\\n=\\n\\nconv(a,b);\\n\\n0.000748 seconds\\n\\n(164\\n\\nallocations:\\n\\n259.313\\n\\nKiB)\\n\\njulia>\\n\\nnorm(ctoep\\n\\n-\\n\\ncconv)\\n\\n2.4593600404835336e-12\\n\\n\\x0c',\n",
       " 'Chapter 8\\n\\nLinear equations\\n\\n8.1\\n\\nLinear and aﬃne functions\\n\\nMatrix-vector product\\n\\nfunction.\\n\\nLet’s deﬁne an instance of\\n\\nthe matrix-vector\\n\\nproduct function, and then numerically check that superpoisition holds.\\n\\n-1.6;\\n\\n2.3\\n\\n-0.6\\n\\n-3.6]\\n\\n#\\n\\nDefine\\n\\n2x3\\n\\nmatrix\\n\\nA\\n\\njulia>\\nA\\n= [-0.1\\n2.8\\n2×3 Array{Float64,2}:\\n\\n-0.1\\n\\n2.8\\n\\n2.3\\n\\n-0.6\\n\\n-1.6\\n\\n-3.6\\n\\njulia>\\n\\nf(x)\\n\\n= A*x\\n\\n#\\n\\nDefine\\n\\nmatrix-vector\\n\\nproduct\\n\\nfunction\\n\\nf (generic function\\n\\nwith\\n\\n1\\n\\nmethod)\\n\\njulia>\\n\\njulia>\\n\\n#\\n\\nx\\n\\nLet’s\\n\\ncheck\\n\\nsuperposition\\n\\n= [1,\\n\\n2,\\n\\n3];\\n\\ny\\n\\njulia>\\n\\nalpha =\\n\\n0.5;\\n\\nbeta\\n\\n=\\n\\n=\\n\\n[-3,\\n\\n-1,\\n\\n2];\\n\\n-1.6;\\n\\njulia>\\n\\nlhs = f(alpha*x+beta*y)\\n\\n2-element Array{Float64,1}:\\n\\n9.47\\n\\n16.75\\n\\njulia>\\n\\nrhs = alpha*f(x)+beta*f(y)\\n\\n2-element Array{Float64,1}:\\n\\n9.47\\n\\n16.75\\n\\njulia>\\n\\nnorm(lhs-rhs)\\n\\n1.7763568394002505e-15\\n\\njulia>\\n\\nf([0,1,0])\\n\\n#\\n\\nShould\\n\\nbe\\n\\nsecond\\n\\ncolumn\\n\\nof\\n\\nA\\n\\n2-element Array{Float64,1}:\\n\\n2.8\\n\\n-0.6\\n\\n\\x0c',\n",
       " '68\\n\\n8\\n\\nLinear equations\\n\\nDe-meaning matrix.\\n\\nLet’s create a de-meaning matrix, and check that it works\\n\\non a vector.\\n\\njulia> de_mean(n)\\n\\n=\\n\\neye(n)\\n\\n.-\\n\\n1/n;\\n\\n#\\n\\nDe-meaning matrix\\n\\njulia> x\\n\\n=\\n\\n[0.2,\\n\\n2.3,\\n\\n1.0];\\n\\njulia> de_mean(length(x))*x\\n\\n#\\n\\nDe-mean\\n\\nusing\\n\\nmatrix\\n\\nmultiplication\\n\\n3-element\\n\\nArray{Float64,1}:\\n\\n-0.966667\\n\\n1.13333\\n\\n-0.166667\\n\\njulia> x\\n\\n.- avg(x)\\n\\n#\\n\\nDe-mean\\n\\nby\\n\\nsubtracting\\n\\nmean\\n\\n3-element\\n\\nArray{Float64,1}:\\n\\n-0.966667\\n\\n1.13333\\n\\n-0.166667\\n\\nExamples of\\n\\nfunctions\\n\\nthat are not\\n\\nlinear.\\n\\nThe componentwise absolute value\\n\\nand the\\n\\nsort\\n\\nfunction are\\n\\nexamples of nonlinear\\n\\nfunctions.\\n\\nThese\\n\\nfunctions are\\n\\neasily computed by abs and sort. By default, the sort function sorts in increasing\\n\\norder, but this can be changed by adding an optional keyword argument.\\n\\njulia> f(x) =\\n\\nabs.(x)\\n\\n#\\n\\ncomponentwise\\n\\nabsolute\\n\\nvalue\\n\\nf\\n\\n(generic\\n\\nfunction\\n\\nwith\\n\\n1\\n\\nmethod)\\n\\njulia> x\\n\\n=\\n\\n[1,\\n\\n0];\\n\\ny\\n\\n=\\n\\n[0,\\n\\n1];\\n\\nalpha\\n\\n=\\n\\n-1;\\n\\nbeta\\n\\n= 2;\\n\\njulia> f(alpha*x\\n\\n+\\n\\nbeta*y)\\n\\n2-element\\n\\nArray{Int64,1}:\\n\\n1\\n\\n2\\n\\njulia> alpha*f(x)\\n\\n+\\n\\nbeta*f(y)\\n\\n2-element\\n\\nArray{Int64,1}:\\n\\n-1\\n\\n2\\n\\njulia>\\n\\nf(x) =\\n\\nsort(x,\\n\\nrev\\n\\n=\\n\\ntrue)\\n\\n#\\n\\nsort\\n\\nin\\n\\ndecreasing\\n\\norder\\n\\nf\\n\\n(generic function\\n\\nwith\\n\\n1\\n\\nmethod)\\n\\njulia>\\n\\nf(alpha*x\\n\\n+\\n\\nbeta*y)\\n\\n2-element\\n\\nArray{Int64,1}:\\n\\n2\\n\\n-1\\n\\njulia> alpha*f(x)\\n\\n+\\n\\nbeta*f(y)\\n\\n2-element\\n\\nArray{Int64,1}:\\n\\n1\\n\\n0\\n\\n\\x0c',\n",
       " \"8.2\\n\\nLinear function models\\n\\n69\\n\\n8.2\\n\\nLinear function models\\n\\nPrice elasticity of demand.\\n\\nLet’s use a price elasticity of demand matrix to pre-\\n\\ndict the demand for three products when the prices are changed a bit. Using this\\n\\nwe can predict the change in total proﬁt, given the manufacturing costs.\\n\\njulia>\\n\\np\\n\\n= [10,\\n\\n20,\\n\\n15];\\n\\n#\\n\\nCurrent\\n\\nprices\\n\\njulia>\\n\\nd\\n\\n= [5.6,\\n\\n1.5,\\n\\n8.6];\\n\\n#\\n\\nCurrent\\n\\ndemand\\n\\n(say\\n\\nin\\n\\nthousands)\\n\\njulia>\\n\\nc\\n\\n= [6.5,\\n\\n11.2,\\n9.8];\\n(p-c)'*d\\n\\n#\\n\\nCost\\n\\nto\\n\\nmanufacture\\n\\njulia>\\n\\nprofit =\\n\\n#\\n\\nCurrent\\n\\ntotal\\n\\nprofit\\n\\n77.51999999999998\\n\\njulia> #\\n\\nDemand\\n\\nelasticity\\n\\nmatrix\\n\\njulia> E\\n3×3\\n\\n=\\n\\n[-0.3\\n\\n0.1\\n\\n-0.1;\\n\\n0.1\\n\\n-0.5\\n\\n0.05\\n\\n;\\n\\n-0.1\\n\\n0.05\\n\\n-0.4]\\n\\nArray{Float64,2}:\\n\\n-0.3\\n\\n0.1\\n\\n-0.1\\n\\n0.1\\n\\n-0.5\\n\\n0.05\\n\\n-0.1\\n\\n0.05\\n\\n-0.4\\n\\njulia> p_new =\\n\\n[9,\\n\\n21,\\n\\n14];\\n\\n#\\n\\nProposed\\n\\nnew\\n\\nprices\\n\\njulia> delta_p\\n\\n=\\n\\n(p_new-p)./p\\n\\n#\\n\\nFractional\\n\\nchange\\n\\nin\\n\\nprices\\n\\n3-element\\n\\nArray{Float64,1}:\\n\\n-0.1\\n\\n0.05\\n\\n-0.0666667\\n\\njulia> delta_d\\n\\n=\\n\\nE*delta_p\\n\\n#\\n\\nPredicted\\n\\nfractional\\n\\nchange\\n\\nin\\n\\ndemand\\n\\n3-element\\n\\nArray{Float64,1}:\\n\\n0.0416667\\n\\n-0.0383333\\n\\n0.0391667\\n\\njulia> d_new =\\n\\nd\\n\\n.*\\n\\n(1\\n\\n.+\\n\\ndelta_d)\\n\\n#\\n\\nPredicted\\n\\nnew\\n\\ndemand\\n\\n3-element\\n\\nArray{Float64,1}:\\n\\n5.833333333333333\\n\\n1.4425\\n\\n8.936833333333333\\n\\njulia> profit_new\\n\\n=\\n\\n(p_new-c)'*d_new\\n\\n#\\n\\nPredicted\\n\\nnew\\n\\nprofit\\n\\n66.25453333333333\\n\\nIf we trust\\n\\nthe linear demand elasticity model, we should not make these price\\n\\nchanges.\\n\\nTaylor approximation.\\n\\nConsider the nonlinear function f : R2 → R2 given by\\n(cid:34) (cid:112)(x1 − a1 )2 + (x2 − a2 )2\\n(cid:112)(x1 − b1 )2 + (x2 − b2 )2\\n\\nf (x) =\\n\\n(cid:34) (cid:107)x − a(cid:107)\\n(cid:107)x − b(cid:107)\\n\\n(cid:35)\\n\\n=\\n\\n(cid:35)\\n\\n.\\n\\n\\x0c\",\n",
       " \"70\\n\\n8\\n\\nLinear equations\\n\\nThe two components of f give the distance of x to the points a and b. The function\\n\\nis diﬀerentiable, except when x = a or x = b.\\n\\nIts derivative or Jacobian matrix is\\n\\n\\uf8ee\\uf8ef\\uf8ef\\uf8f0 ∂ f1\\n\\n∂x1\\n\\n∂ f2\\n\\n∂x1\\n\\n(z )\\n\\n(z )\\n\\n∂ f1\\n\\n∂x2\\n\\n∂ f2\\n\\n∂x2\\n\\n(z )\\n\\n(z )\\n\\n\\uf8f9\\uf8fa\\uf8fa\\uf8fb =\\n\\n\\uf8ee\\uf8ef\\uf8ef\\uf8f0\\n\\ngiven by\\n\\nDf (z ) =\\n\\n\\uf8f9\\uf8fa\\uf8fa\\uf8fb .\\n\\nz1 − a1\\n\\n(cid:107)z − a(cid:107)\\n\\nz1 − b1\\n\\n(cid:107)z − b(cid:107)\\n\\nz2 − a2\\n\\n(cid:107)z − a(cid:107)\\n\\nz2 − b2\\n\\n(cid:107)z − b(cid:107)\\n\\nLet’s form the Taylor approximation of f\\n\\nfor some speciﬁc values of a, b, and z ,\\n\\nand then check it against the true value of f at a few points near z .\\n\\njulia> f(x) =\\n\\n[\\n\\njulia> Df(z) =\\n\\n[\\n\\nnorm(x-a),\\n(z-a)' /\\n\\nnorm(x-b)\\n\\n];\\n\\nnorm(z-a)\\n\\n;\\n\\n(z-b)'\\n\\n/\\n\\nnorm(z-b)\\n\\n];\\n\\njulia> f_hat(x)\\n\\n= f(z)\\n\\n+\\n\\nDf(z)*(x-z);\\n\\njulia> a\\n\\n=\\n\\n[1,\\n\\n0];\\n\\nb\\n\\n=\\n\\n[1,\\n\\n1];\\n\\nz\\n\\n=\\n\\n[0,\\n\\n0];\\n\\njulia> f([0.1,\\n\\n0.1])\\n\\n2-element\\n\\nArray{Float64,1}:\\n\\n0.905539\\n\\n1.27279\\n\\njulia> f_hat([0.1,\\n\\n0.1])\\n\\n2-element\\n\\nArray{Float64,1}:\\n\\n0.9\\n\\n1.27279\\n\\njulia> f([0.5,\\n\\n0.5])\\n\\n2-element\\n\\nArray{Float64,1}:\\n\\n0.707107\\n\\n0.707107\\n\\njulia> f_hat([0.5,\\n\\n0.5])\\n\\n2-element\\n\\nArray{Float64,1}:\\n\\n0.5\\n\\n0.707107\\n\\nRegression model. We revisit\\n\\nthe regression model\\n\\nfor\\n\\nthe house sales data in\\n\\nSection 2.3. The model\\n\\nis\\n\\nˆy = xT β + v = β1x1 + β2x2 + v ,\\n\\nwhere ˆy is the predicted house sale price, x1 is the house area in 1000 square feet,\\nand x2 is the number of bedrooms.\\n\\nIn the following code we construct\\noutcomes yd ,\\nfor\\nthe N = 774 examples\\nin the data set. We then calculate the\\nregression model predictions ˆyd , the prediction errors rd , and the RMS prediction\\n\\nthe 2 × 774 data matrix X and vector of\\n\\nerror.\\n\\n\\x0c\",\n",
       " '8.3\\n\\nSystems of\\n\\nlinear equations\\n\\n71\\n\\njulia>\\n\\n#\\n\\nparameters\\n\\nin\\n\\nregression\\n\\nmodel\\n\\njulia>\\n\\nbeta\\n\\n= [148.73,\\n\\n-18.85];\\n\\nv\\n\\n=\\n\\n54.40;\\n\\njulia>\\n\\nD\\n\\n= house_sales_data();\\n\\njulia>\\n\\nyd =\\n\\nD[\"price\"];\\n\\n#\\n\\nvector\\n\\nof\\n\\noutcomes\\n\\njulia>\\n\\nN\\n\\n= length(yd)\\n\\n774\\n\\njulia>\\n\\nX\\n\\n= [ D[\"area\"]\\n\\nD[\"beds\"]\\n\\n]\\';\\n\\njulia> size(X)\\n\\n(2, 774)\\n\\njulia> ydhat =\\n\\nX\\'*beta\\n\\n.+ v;\\n\\n#\\n\\nvector\\n\\nof\\n\\npredicted\\n\\noutcomes\\n\\njulia>\\n\\nrd =\\n\\nyd\\n\\n-\\n\\nydhat;\\n\\n#\\n\\nvector\\n\\nof\\n\\npredicted\\n\\nerrors\\n\\njulia>\\n\\nrms(rd)\\n\\n#\\n\\nRMS\\n\\nprediction\\n\\nerror\\n\\n74.84571862623022\\n\\njulia>\\n\\n# Compare\\n\\nwith\\n\\nstandard\\n\\ndeviation\\n\\nof\\n\\nprices\\n\\njulia>\\n\\nstdev(yd)\\n\\n112.7821615975651\\n\\n8.3\\n\\nSystems of\\n\\nlinear equations\\n\\nBalancing chemical reactions. We verify the linear balancing equations on page 155\\n\\nof VMLS,\\n\\nfor the simple example of electrolysis of water.\\n\\njulia>\\n\\nR\\n\\n= [2 ;\\n\\n1]\\n\\n2-element Array{Int64,1}:\\n\\n2\\n\\n1\\n\\njulia>\\nP\\n= [2 0\\n;\\n0\\n2×2 Array{Int64,2}:\\n\\n2]\\n\\n2\\n\\n0\\n\\n0\\n\\n2\\n\\njulia>\\n\\n#\\n\\nCheck\\n\\nbalancing\\n\\ncoefficients\\n\\n[2,2,1]\\n\\njulia>\\n\\ncoeff =\\n\\n[2,2,1];\\n\\njulia>\\n\\n[R -P]*coeff\\n\\n2-element Array{Int64,1}:\\n\\n0\\n\\n0\\n\\n\\x0c',\n",
       " '72\\n\\n8\\n\\nLinear equations\\n\\n\\x0c',\n",
       " 'Chapter 9\\n\\nLinear dynamical systems\\n\\n9.1\\n\\nLinear dynamical systems\\n\\nLet’s simulate a time-invariant linear dynamic system\\n\\nwith dynamics matrix\\n\\nxt+1 = Axt ,\\n\\nt = 1, . . . , T ,\\n\\n\\uf8ee\\uf8f0\\n\\nA =\\n\\n0.97\\n−0.30\\n\\n0.01\\n\\n0.10\\n\\n0.99\\n−0.04\\n\\n−0.05\\n\\n0.05\\n\\n0.96\\n\\n\\uf8f9\\uf8fb\\n\\nand initial state x1 = (1, 0, −1). We store the state tra jectory in the n × T matrix\\nstate_traj, with the ith column xt . We plot the result in Figure 9.1.\\n\\njulia>\\n\\nx_1 = [1,0,-1];\\n\\n#\\n\\ninitial\\n\\nstate\\n\\njulia>\\n\\nn\\n\\n= length(x_1);\\n\\nT\\n\\n=\\n\\n50;\\n\\n-0.05\\n\\n;\\n\\n-0.3\\n\\n0.99\\n\\n0.05\\n\\n;\\n\\n0.01\\n\\n-0.04\\n\\n0.96\\n\\n]\\n\\njulia>\\nA\\n= [ 0.97\\n0.10\\n3×3 Array{Float64,2}:\\n\\n0.97\\n\\n-0.3\\n\\n0.1\\n\\n0.99\\n\\n0.01\\n\\n-0.04\\n\\n-0.05\\n\\n0.05\\n\\n0.96\\n\\njulia>\\n\\nstate_traj\\n\\n=\\n\\n[x_1\\n\\nzeros(n,T-1)\\n\\n];\\n\\njulia>\\n\\nfor t=1:T-1\\n\\n#\\n\\nDynamics\\n\\nrecursion\\n\\nstate_traj[:,t+1]\\n\\n=\\n\\nA*state_traj[:,t];\\n\\nend\\n\\njulia> using Plots\\nstate_traj\\',\\n\\njulia> plot(1:T,\\n\\nxlabel\\n\\n=\\n\\n\"t\",\\n\\nlabel\\n\\n= [\"(x_t)_1\",\\n\\n\"(x_t)_2\",\\n\\n\"(x_t)_3\"])\\n\\n\\x0c',\n",
       " '74\\n\\n9\\n\\nLinear dynamical systems\\n\\nFigure 9.1 Linear dynamical system simulation.\\n\\n9.2\\n\\nPopulation dynamics\\n\\nWe\\n\\ncan create\\n\\na population dynamics matrix with just\\n\\none\\n\\nsimple\\n\\nline\\n\\nof Ju-\\n\\nlia. The following code predicts the 2020 population distribution in the US using\\n\\nthe data of Section 9.2 of VMLS, which are available through the VMLS function\\n\\npopulation_data. The result is shown in Figure 9.2.\\n\\njulia> #\\n\\nImport\\n\\n3\\n\\n100-vectors:\\n\\npopulation,\\n\\nbirth_rate,\\n\\ndeath_rate\\n\\njulia> D\\n\\njulia> b\\n\\njulia> d\\n\\njulia> A\\n\\njulia> x\\n\\n=\\n\\n=\\n\\n=\\n\\n=\\n\\n=\\n\\npopulation_data();\\n\\nD[\"birth_rate\"];\\n\\nD[\"death_rate\"];\\n[b\\';\\n\\ndiagonal(1 .-\\n\\nD[\"population\"];\\n\\njulia> for\\n\\nk =\\n\\n1:10\\n\\nglobal x\\n\\nx\\n\\n= A*x;\\n\\nend;\\n\\njulia> using Plots\\n\\nd[1:end-1])\\n\\nzeros(length(d)-1)];\\n\\njulia> plot(x,\\n\\nlegend=false,\\n\\nxlabel\\n\\n=\\n\\n\"Age\",\\n\\nylabel\\n\\n= \"Population\\n\\n(millions)\")\\n\\nNote the keyword global in the for-loop. Without this statement, the scope of the\\n\\nvariable x created by the assignment x\\n\\n=\\n\\nA*x would be local to the for-loop,\\n\\ni.e.,\\n\\nthis variable does not exist outside the loop and is diﬀerent from the x outside the\\n\\nloop.\\n\\n01020304050-2-1012t(x_t)_1(x_t)_2(x_t)_3\\x0c',\n",
       " '9.3\\n\\nEpidemic dynamics\\n\\n75\\n\\nFigure 9.2 Predicted age distribution in the US in 2020.\\n\\n9.3\\n\\nEpidemic dynamics\\n\\nLet’s implement\\n\\nthe simulation of\\n\\nthe epidemic dynamics from VMLS §9.3. The\\n\\nplot is in ﬁgure 9.3.\\n\\njulia>\\n\\njulia>\\n\\nT\\n\\nA\\n\\n= 210;\\n\\n= [ 0.95\\n\\n0.04\\n\\n0\\n\\n0\\n\\n0\\n\\n0.10\\n\\n1 0\\n\\n;\\n\\n;\\n\\n0.05\\n\\n0.85\\n\\n0\\n\\n0.01\\n\\n0\\n\\n0\\n\\n0\\n\\n1\\n\\n;\\n\\n];\\n\\njulia>\\n\\nx_1 =\\n\\n[1,0,0,0];\\n\\njulia>\\n\\nstate_traj\\n\\n=\\n\\n[x_1\\n\\nzeros(4,T-1)\\n\\n];\\n\\n#\\n\\nState trajectory\\n\\njulia>\\n\\nfor t=1:T-1\\n\\n#\\n\\nDynamics\\n\\nrecursion\\n\\nstate_traj[:,t+1]\\n\\n=\\n\\nA*state_traj[:,t];\\n\\nend\\n\\njulia>\\n\\nusing\\n\\njulia>\\n\\nplot(1:T,\\n\\nPlots\\nstate_traj\\',\\n\\nxlabel\\n\\n=\\n\\n\"Time\\n\\nt\",\\n\\nlabel\\n\\n= [\"Susceptible\",\\n\\n\"Infected\",\\n\\n\"Recovered\", \"Deceased\"])\\n\\n9.4\\n\\nMotion of a mass\\n\\nLet’s simulate the discretized model of the motion of a mass in §9.4 of VMLS. See\\n\\nﬁgure 9.4.\\n\\n025507510001234AgePopulation (millions)\\x0c',\n",
       " '76\\n\\n9\\n\\nLinear dynamical systems\\n\\nFigure 9.3 Simulation of epidemic dynamics.\\n\\njulia> h\\n\\njulia> A\\n\\njulia> B\\n\\n=\\n\\n=\\n\\n=\\n\\n0.01;\\n\\n[ 1\\n\\n[ 0\\n\\nh\\n\\n;\\n\\nm\\n\\n;\\n\\n=\\n\\n0\\n\\n1;\\n\\neta\\n\\n=\\n\\n1;\\n\\n1-h*eta/m\\n\\n];\\n\\nh/m\\n\\n];\\n\\njulia> x1\\n\\n= [0,0];\\n\\njulia> K\\n\\njulia> f\\n\\njulia> X\\n\\n=\\n\\n=\\n\\n=\\n\\n600;\\n\\n#\\n\\nsimulate\\n\\nfor\\n\\nK*h\\n\\n=\\n\\n6\\n\\nseconds\\n\\nzeros(K);\\n\\nf[50:99]\\n\\n.=\\n\\n1.0;\\n\\nf[100:139] .=\\n\\n-1.3;\\n\\n[x1\\n\\nzeros(2,K-1)];\\n\\njulia> for\\n\\nk=1:K-1\\n\\nX[:,k+1] = A*\\n\\nX[:,k]\\n\\n+\\n\\nB*f[k]\\n\\nend\\n\\njulia> using Plots\\n\\njulia> plot(X[1,:],\\n\\nxlabel=\"k\",\\n\\nylabel=\"Position\",\\n\\nlegend=false\\n\\njulia> plot(X[2,:],\\n\\nxlabel=\"k\",\\n\\nylabel=\"Velocity\",\\n\\nlegend=false\\n\\n)\\n\\n)\\n\\n9.5\\n\\nSupply chain dynamics\\n\\n0501001502000.000.250.500.751.00Time tSusceptibleInfectedRecoveredDeceased\\x0c',\n",
       " '9.5\\n\\nSupply chain dynamics\\n\\n77\\n\\nFigure 9.4 Simulation of a mass moving along a line:\\n\\nposition (top) and\\n\\nvelocity (bottom).\\n\\n01002003004005006000.000.050.100.15kPosition0100200300400500600-0.10.00.10.20.30.4kVelocity\\x0c',\n",
       " '78\\n\\n9\\n\\nLinear dynamical systems\\n\\n\\x0c',\n",
       " \"Chapter 10\\n\\nMatrix multiplication\\n\\n10.1\\n\\nMatrix-matrix multiplication\\n\\nIn Julia the product of matrices A and B is obtained with A*B. We calculate the\\n\\nmatrix product on page 177 of VMLS.\\n\\njulia>\\nA\\n= [-1.5\\n3\\n2;\\n2×3 Array{Float64,2}:\\n\\n1\\n\\n-1\\n\\n0]\\n\\n-1.5\\n\\n3.0\\n\\n1.0\\n\\n-1.0\\n\\n2.0\\n\\n0.0\\n\\njulia>\\nB\\n= [-1\\n-1;\\n3×2 Array{Int64,2}:\\n\\n0\\n\\n-2;\\n\\n1\\n\\n0]\\n\\n-1\\n\\n0\\n\\n1\\n\\n-1\\n\\n-2\\n\\n0\\n\\njulia>\\nC\\n= A*B\\n2×2 Array{Float64,2}:\\n\\n3.5\\n\\n-4.5\\n\\n-1.0\\n\\n1.0\\n\\nGram matrix.\\n\\nThe Gram matrix of a matrix A is the matrix G = AT A.\\n\\nIt is a\\n\\nsymmetric matrix and the i, j element Gij\\nof A.\\n\\nis the inner product of columns i and j\\n\\njulia>\\n\\njulia>\\n3×3\\n\\nA\\n\\nG\\n\\n= randn(10,3);\\n= A'*A\\n\\nArray{Float64,2}:\\n\\n11.1364\\n\\n-3.91865\\n\\n-3.69057\\n\\n-3.91865\\n\\n7.98358\\n\\n2.4839\\n\\n-3.69057\\n\\n2.4839\\n\\n10.956\\n\\n\\x0c\",\n",
       " \"80\\n\\n10\\n\\nMatrix multiplication\\n\\njulia> #\\n\\nGii is\\n\\nnorm\\n\\nof\\n\\ncolumn\\n\\ni,\\n\\nsquared\\n\\njulia> G[2,2]\\n\\n7.983579175590987\\n\\njulia> norm(A[:,2])^2\\n\\n7.983579175590987\\n\\njulia> #\\n\\nGij is\\n\\ninner\\n\\nproduct\\n\\nof\\n\\ncolumns\\n\\ni\\n\\nand\\n\\nj\\n\\njulia> G[1,3]\\n\\n-3.6905664879621454\\njulia> A[:,1]'*A[:,3]\\n\\n-3.6905664879621463\\n\\nComplexity of matrix triple product.\\n\\nLet’s check the associative property, which\\nstates that (AB )C = A(BC ) for any m × n matrix A, any n × p matrix B , and any\\np × q matrix B . At the same time we will see that the left-hand and right-hand\\n\\nsides take very diﬀerent amounts of time to compute.\\n\\njulia> m\\n\\n=\\n\\n2000;\\n\\nn\\n\\n=\\n\\n50;\\n\\nq\\n\\n=\\n\\n2000;\\n\\np\\n\\n=\\n\\n2000;\\n\\njulia> A\\n\\n=\\n\\nrandn(m,n);\\n\\nB\\n\\n=\\n\\nrandn(n,p);\\n\\nC\\n\\n=\\n\\nrandn(p,q);\\n\\njulia> @time LHS\\n\\n=\\n\\n(A*B)*C;\\n\\n0.819912\\n\\nseconds\\n\\n(245.32\\n\\nk\\n\\nallocations:\\n\\n72.557\\n\\nMiB,\\n\\n13.47%\\n\\ngc\\n\\ntime)\\n\\njulia> @time LHS\\n\\n=\\n\\n(A*B)*C;\\n\\n0.254107\\n\\nseconds\\n\\n(8\\n\\nallocations:\\n\\n61.035\\n\\nMiB,\\n\\n20.85%\\n\\ngc\\n\\ntime)\\n\\njulia> @time RHS\\n\\n=\\n\\nA*(B*C);\\n\\n0.030907\\n\\nseconds\\n\\n(9\\n\\nallocations:\\n\\n31.281\\n\\nMiB,\\n\\n13.95%\\n\\ngc\\n\\ntime)\\n\\njulia> @time RHS\\n\\n=\\n\\nA*(B*C);\\n\\n0.023507\\n\\nseconds\\n\\n(8\\n\\nallocations:\\n\\n31.281\\n\\nMiB,\\n\\n3.13%\\n\\ngc\\n\\ntime)\\n\\njulia> norm(LHS-RHS)\\n\\n5.334805188873507e-10\\n\\njulia> @time D\\n\\n=\\n\\nA*B*C;\\n\\n#\\n\\nevaluated\\n\\nas\\n\\n(A*B)*C\\n\\nor\\n\\nas\\n\\nA*(B*C)?\\n\\n0.220616\\n\\nseconds\\n\\n(1.03\\n\\nk\\n\\nallocations:\\n\\n61.098\\n\\nMiB, 5.09%\\n\\ngc\\n\\ntime)\\n\\nWe see that evaluating (A*B)*C takes around 10 times as much time as evaluating\\n\\nA*(B*C), which is predicted from the complexities.\\n\\nIn the last line we deduce that\\n\\nA*B*C is evaluated left to right, as (A*B)*C. Note that for these particular matrices,\\n\\nthis is the (much) slower order to multiply the matrices.\\n\\n10.2\\n\\nComposition of\\n\\nlinear functions\\n\\nSecond diﬀerence matrix. We compute the second diﬀerence matrix on page 184\\n\\nof VMLS.\\n\\n\\x0c\",\n",
       " '10.3\\n\\nMatrix power\\n\\n81\\n\\njulia>\\n\\nD(n)\\n\\n= [-eye(n-1)\\n\\nzeros(n-1)]\\n\\n+\\n\\n[zeros(n-1)\\n\\neye(n-1)];\\n\\njulia>\\nD(5)\\n4×5 Array{Float64,2}:\\n\\n-1.0\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n-1.0\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n-1.0\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n-1.0\\n\\n1.0\\n\\njulia>\\nD(4)\\n3×4 Array{Float64,2}:\\n\\n-1.0\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n-1.0\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n-1.0\\n\\n1.0\\n\\njulia>\\nDelta =\\nD(4)*D(5)\\n3×5 Array{Float64,2}:\\n\\n#\\n\\nSecond\\n\\ndifference\\n\\nmatrix\\n\\n1.0\\n\\n-2.0\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n1.0\\n\\n-2.0\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n1.0\\n\\n-2.0\\n\\n1.0\\n\\n10.3\\n\\nMatrix power\\n\\nThe kth power of a square matrix A is denoted Ak .\\n\\nIn Julia, this power is formed\\n\\nusing A^k.\\n\\nLet’s form the adjacency matrix of the directed graph on VMLS page 186. Then\\n\\nlet’s ﬁnd out how many cycles of\\n\\nlength 8 there are, starting from each node.\\n\\n(A\\n\\ncycle is a path that starts and stops at the same node.)\\n\\njulia>\\nA\\n= [ 0\\n1\\n0\\n5×5 Array{Int64,2}:\\n\\n0\\n\\n1;\\n\\n1\\n\\n0\\n\\n1\\n\\n0\\n\\n0;\\n\\n0\\n\\n0\\n\\n1\\n\\n1\\n\\n1;\\n\\n1\\n\\n0\\n\\n0\\n\\n0\\n\\n0;\\n\\n0\\n\\n0\\n\\n0\\n\\n1\\n\\n0]\\n\\n0\\n\\n1\\n\\n0\\n\\n0\\n\\n1\\n\\n1\\n\\n0\\n\\n1\\n\\n0\\n\\n0\\n\\n0\\n\\n0\\n\\n1\\n\\n1\\n\\n1\\n\\n1\\n\\n0\\n\\n0\\n\\n0\\n\\n0\\n\\n0\\n\\n0\\n\\n0\\n\\n1\\n\\n0\\n\\njulia>\\nA^2\\n5×5 Array{Int64,2}:\\n\\n1\\n\\n0\\n\\n1\\n\\n1\\n\\n0\\n\\n0\\n\\n1\\n\\n1\\n\\n1\\n\\n2\\n\\n1\\n\\n0\\n\\n1\\n\\n2\\n\\n1\\n\\n0\\n\\n1\\n\\n0\\n\\n0\\n\\n1\\n\\n1\\n\\n0\\n\\n0\\n\\n0\\n\\n0\\n\\n\\x0c',\n",
       " '82\\n\\n10\\n\\nMatrix multiplication\\n\\njulia> A^8\\n5×5\\n\\nArray{Int64,2}:\\n\\n18\\n\\n11\\n\\n15\\n\\n20\\n\\n20\\n\\n25\\n\\n14\\n\\n21\\n\\n28\\n\\n26\\n\\n24\\n\\n14\\n\\n20\\n\\n27\\n\\n26\\n\\n11\\n\\n6\\n\\n9\\n\\n12\\n\\n11\\n\\n6\\n\\n4\\n\\n5\\n\\n7\\n\\n7\\n\\njulia>\\n\\nnumber_of_cycles\\n\\n=\\n\\ndiag(A^8)\\n\\n5-element\\n\\nArray{Int64,1}:\\n\\n18\\n\\n14\\n\\n20\\n\\n12\\n\\n7\\n\\nPopulation dynamics.\\n\\nLet’s write the code for ﬁgure 10.2 in VMLS, which plots\\n\\nthe contribution factor to the total US population in 2020 (ignoring immigration),\\n\\nfor\\n\\neach age\\n\\nin 2010.\\n\\nThe Julia plot\\n\\nis\\n\\nin ﬁgure 10.1. We\\n\\ncan see\\n\\nthat, not\\n\\nsurprisingly, 20–25 year olds have the highest contributing factor, around 1.5. This\\n\\nmeans that on average, each 20-25 year old in 2010 will be responsible for around\\n\\n1.5 people in 2020.\\n\\nThis\\n\\ntakes\\n\\ninto account any children they may have before\\n\\nthen, and (as a very small eﬀect) the few of them who will no longer be with us in\\n\\n2020.\\n\\njulia> D\\n\\n=\\n\\npopulation_data();\\n\\njulia> b\\n\\n=\\n\\nD[\"birth_rate\"];\\n\\njulia> d\\n\\n=\\n\\nD[\"death_rate\"];\\n\\njulia> #\\n\\nDynamics\\n[b\\';\\n\\nmatrix\\n\\nfor\\n\\npopulaion\\n\\ndynamics\\n\\njulia> A\\n\\n=\\n\\ndiagonal(1 .-\\n\\nd[1:end-1])\\n\\nzeros(length(d)-1)];\\n\\njulia> #\\n\\nContribution\\n\\nfactor\\n\\nto\\n\\ntotal\\n\\npoulation\\n\\nin\\n\\n2020\\n\\njulia> #\\n\\nfrom\\neach\\nage\\nin\\n2010\\n= ones(100)\\'*(A^10);\\n\\njulia> cf\\n\\n#\\n\\nContribution\\n\\nfactor\\n\\njulia>\\n\\nusing\\nPlots\\nplot(cf\\',\\n\\njulia>\\n\\nlegend =\\n\\nfalse,\\n\\nxlabel\\n\\n=\\n\\n\"Age\",\\n\\nylabel\\n\\n=\\n\\n\"Factor\")\\n\\n10.4\\n\\nQR factorization\\n\\nIn Julia,\\n\\nthe QR factorization of a matrix A can be\\n\\nfound using qr(A), which\\n\\nreturns a tuple with the Q and R factors. However the matrix Q is not returned as\\n\\n\\x0c',\n",
       " '10.4\\n\\nQR factorization\\n\\n83\\n\\nFigure 10.1 Contribution factor per age in 2010 to the total population in\\n2020. The value for age i − 1 is the ith component of the row vector 1T A10 .\\n\\nan array, but in a special compact format.\\n\\nIt can be converted to a regular matrix\\n\\nvariable using the command Matrix(Q). Hence, the QR factorization as deﬁned in\\n\\nVMLS is computed by a sequence of two commands:\\n\\njulia>\\n\\nQ, R\\n\\n= qr(A);\\n\\njulia>\\n\\nQ\\n\\n= Matrix(Q);\\n\\nThe following example also illustates a second, but minor diﬀerence with the\\n\\nVMLS deﬁnition. The R factor computed by Julia may have negative elements on\\n\\nthe diagonal, as opposed to only positive elements if we follow the deﬁnition used\\n\\nin VMLS. The two deﬁnitions are equivalent, because if Rii\\nchange the sign of the ith row of R and the ith column of Q, to get an equivalent\\n\\nis negative, one can\\n\\nfactorization with Rii > 0.\\nnegative elements on the diagonal do not pose any problem in applications of the\\n\\nstep is not needed in practice,\\n\\nsince\\n\\nHowever\\n\\nthis\\n\\nQR factorization.\\n\\njulia>\\n\\nA\\n\\n= randn(6,4);\\n\\njulia>\\n\\nQ, R\\n\\n= qr(A);\\n\\njulia>\\nR\\n4×4 Array{Float64,2}:\\n\\n1.36483\\n\\n-1.21281\\n\\n-0.470052\\n\\n-1.40011\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n-2.01191\\n\\n-2.92458\\n\\n-0.802368\\n\\n0.0\\n\\n0.0\\n\\n-1.94759\\n\\n0.0\\n\\n0.84228\\n\\n1.19766\\n\\njulia>\\nQ\\n= Matrix(Q)\\n6×4 Array{Float64,2}:\\n\\n02550751000.00.51.01.5AgeFactor\\x0c',\n",
       " \"84\\n\\n10\\n\\nMatrix multiplication\\n\\n-0.454997\\n\\n-0.577229\\n\\n-0.442971\\n\\n-0.00869016\\n\\n0.121067\\n\\n-0.301916\\n\\n0.349002\\n\\n-0.370212\\n\\n0.00581297\\n\\n-0.159604\\n\\n-0.647797\\n\\n0.388812\\n\\n0.223813\\n\\n0.546873\\n\\n-0.507827\\n\\n0.28373\\n\\n-0.46152\\n\\n-0.30155\\n\\n0.221266\\n\\n-0.550089\\n\\n-0.328929\\n\\n0.48673\\n\\n0.387945\\n\\n0.681065\\n\\njulia> norm(Q*R-A)\\n\\n1.0046789954275695e-15\\njulia> Q'*Q\\n4×4\\n\\nArray{Float64,2}:\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n1.0\\n\\n-2.77556e-17\\n\\n2.77556e-17\\n\\n-8.32667e-17\\n\\n-1.66533e-16\\n\\n-2.77556e-17\\n\\n-8.32667e-17\\n\\n1.0\\n\\n5.55112e-17\\n\\n2.77556e-17\\n\\n-1.66533e-16\\n\\n5.55112e-17\\n\\n1.0\\n\\n\\x0c\",\n",
       " 'Chapter 11\\n\\nMatrix inverses\\n\\n11.1\\n\\nLeft and right inverses\\n\\nWe’ll see later how to ﬁnd a left or right inverse, when one exists.\\n\\njulia>\\nA\\n= [-3\\n-4;\\n3×2 Array{Int64,2}:\\n\\n4\\n\\n6;\\n\\n1\\n\\n1]\\n\\n-3\\n\\n-4\\n\\n4\\n\\n1\\n\\n6\\n\\n1\\n\\njulia>\\nB\\n= [-11\\n-10\\n16;\\n2×3 Array{Float64,2}:\\n\\n7\\n\\n8\\n\\n-11]/9\\n\\n#\\n\\nA\\n\\nleft\\n\\ninverse\\n\\nof\\n\\nA\\n\\n-1.22222\\n\\n-1.11111\\n\\n1.77778\\n\\n0.777778\\n\\n0.888889\\n\\n-1.22222\\n\\n1\\n\\n-4]/2\\n\\n#\\n\\nAnother\\n\\nleft\\n\\ninverse\\n\\nof\\n\\nA\\n\\njulia>\\nC\\n= [0 -1\\n6;\\n0\\n2×3 Array{Float64,2}:\\n\\n0.0\\n\\n0.0\\n\\n-0.5\\n\\n3.0\\n\\n0.5\\n\\n-2.0\\n\\njulia>\\n\\n#\\n\\nLet’s\\n\\ncheck\\n\\njulia>\\nB*A\\n2×2 Array{Float64,2}:\\n\\n1.0\\n\\n-4.44089e-16\\n\\n0.0\\n\\n1.0\\n\\njulia>\\nC*A\\n2×2 Array{Float64,2}:\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n1.0\\n\\n\\x0c',\n",
       " '86\\n\\n11\\n\\nMatrix inverses\\n\\n11.2\\n\\nInverse\\n\\nIf A is invertible,\\n\\nits inverse is given by inv(A) (and also A^-1). You’ll get an error\\n\\nif A is not invertible, or not square.\\n\\njulia> A\\n3×3\\n\\n=\\n\\n[1\\n\\n-2\\n\\n3;\\n\\n0\\n\\n2\\n\\n2;\\n\\n-4\\n\\n-4\\n\\n-4]\\n\\nArray{Int64,2}:\\n\\n1\\n\\n-2\\n\\n3\\n\\n0\\n\\n2\\n\\n2\\n\\n-4\\n\\n-4\\n\\n-4\\n\\njulia> B\\n3×3\\n\\n=\\n\\ninv(A)\\n\\nArray{Float64,2}:\\n\\n0.0\\n\\n-0.5\\n\\n-0.25\\n\\n-0.2\\n\\n0.2\\n\\n-0.05\\n\\n0.2\\n\\n0.3\\n\\n0.05\\n\\njulia>\\n3×3\\n\\nB*A\\n\\nArray{Float64,2}:\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n1.0\\n\\n2.77556e-17\\n\\n0.0\\n\\n5.55112e-17\\n\\n1.0\\n\\njulia>\\n3×3\\n\\nA*B\\n\\nArray{Float64,2}:\\n\\n1.0\\n\\n1.11022e-16\\n\\n0.0\\n\\n5.55112e-17\\n\\n1.0\\n\\n1.38778e-17\\n\\n-1.11022e-16\\n\\n-2.22045e-16\\n\\n1.0\\n\\nDual basis.\\n\\nThe next example illustrates the dual basis provided by the rows of\\nthe inverse B = A−1 . We calculate the expansion\\n\\n1 x)a1 + · · · + (bT\\nx = (bT\\nn x)an\\n\\nfor a 3 × 3 example (see page 205 of VMLS).\\n\\njulia> A\\n3×3\\n\\n=\\n\\n[ 1\\n\\n0\\n\\n1;\\n\\n4\\n\\n-3\\n\\n-4;\\n\\n1\\n\\n-1\\n\\n-2]\\n\\nArray{Int64,2}:\\n\\n1\\n\\n0\\n\\n1\\n\\n4\\n\\n-3\\n\\n-4\\n\\n1\\n\\n-1\\n\\n-2\\n\\njulia> B\\n3×3\\n\\n=\\n\\ninv(A)\\n\\nArray{Float64,2}:\\n\\n2.0\\n\\n-1.0\\n\\n3.0\\n\\n4.0\\n\\n-3.0\\n\\n8.0\\n\\n-1.0\\n\\n1.0\\n\\n-3.0\\n\\n\\x0c',\n",
       " \"11.3\\n\\nSolving linear equations\\n\\n87\\n\\njulia>\\n\\nx\\n\\n= [ 0.2,\\n\\n-0.3,\\n\\n1.2]\\n\\n3-element Array{Float64,1}:\\n\\n0.2\\n\\n-0.3\\n\\n1.2\\n\\njulia>\\n\\nrhs = (B[1,:]'*x) *\\n(B[3,:]'*x) *\\n\\n+\\n\\nA[:,1]\\n\\n+\\n\\n(B[2,:]'*x)\\n\\n* A[:,2]\\n\\nA[:,3]\\n\\n3-element Array{Float64,1}:\\n\\n0.2\\n\\n-0.3\\n\\n1.2\\n\\nInverse via QR factorization.\\n\\nThe inverse of a matrix A can be computed from\\nits QR factorization A = QR via the formula A−1 = R−1QT .\\n\\njulia>\\n\\nA\\n\\n= randn(3,3);\\n\\njulia>\\ninv(A)\\n3×3 Array{Float64,2}:\\n\\n-0.321679\\n\\n0.323945\\n\\n-0.347063\\n\\n-1.0735\\n\\n-5.03083\\n\\n-3.24503\\n\\n-1.17582\\n\\n-2.68161\\n\\n-1.8496\\n\\njulia>\\n\\nQ, R\\n\\n= qr(A);\\n\\njulia>\\n\\nQ\\n= Matrix(Q);\\ninv(R)*Q'\\njulia>\\n3×3 Array{Float64,2}:\\n\\n-0.321679\\n\\n0.323945\\n\\n-0.347063\\n\\n-1.0735\\n\\n-5.03083\\n\\n-3.24503\\n\\n-1.17582\\n\\n-2.68161\\n\\n-1.8496\\n\\n11.3\\n\\nSolving linear equations\\n\\nBack substitution.\\n\\nLet’s ﬁrst implement back substitution (VMLS Algorithm 11.1)\\n\\nin Julia, and check it. You won’t need this function, since Julia has a better im-\\n\\nplementation of\\n\\nit built in (via the backslash operation discussed below). We give\\n\\nit here only to demonstrate that it works.\\n\\njulia>\\n\\nfunction\\n\\nback_subst(R,b)\\n\\nn\\n\\nx\\n\\n= length(b)\\n\\n= zeros(n)\\n\\n\\x0c\",\n",
       " \"88\\n\\n11\\n\\nMatrix inverses\\n\\nfor i=n:-1:1\\n\\nx[i]\\n\\n= (b[i]\\n\\n-\\n\\nR[i,i+1:n]'*x[i+1:n])\\n\\n/\\n\\nR[i,i]\\n\\nend\\n\\nreturn x\\n\\nend;\\n\\njulia>\\n4×4\\n\\nR = triu(randn(4,4))\\n\\n#\\n\\nRandom\\n\\n4x4\\n\\nupper\\n\\ntriangular\\n\\nmatrix\\n\\nArray{Float64,2}:\\n\\n-0.498881\\n\\n-0.880538\\n\\n-0.745078\\n\\n0.125678\\n\\n0.0\\n\\n-0.922477\\n\\n-0.00673699\\n\\n0.30122\\n\\n0.0\\n\\n0.0\\n\\n-0.283035\\n\\n-0.0184466\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n-2.01396\\n\\njulia> b\\n\\n=\\n\\nrand(4);\\n\\njulia> x\\n\\n=\\n\\nback_subst(R,b);\\n\\njulia> norm(R*x-b)\\n\\n2.220446049250313e-16\\n\\nThe function triu gives the upper triangular part of a matrix, i.e.,\\n\\nit zeros out the\\n\\nentries below the diagonal.\\n\\nBackslash notation.\\n\\nThe Julia command for solving a set of\\n\\nlinear equations\\n\\nAx = b\\n\\nis x=A\\\\b. This\\n\\nis\\n\\nfaster\\n\\nthan x=inv(A)*b, which ﬁrst computes\\n\\nthe inverse of A\\n\\nand then multiplies it with b.\\n\\njulia> n\\n\\n=\\n\\n5000;\\n\\njulia> A\\n\\n=\\n\\nrandn(n,n);\\n\\nb\\n\\n=\\n\\nrandn(n);\\n\\n#\\n\\nrandom\\n\\nset\\n\\nof\\n\\nequations\\n\\njulia> @time x1\\n\\n=\\n\\nA\\\\b;\\n\\n1.422173\\n\\nseconds\\n\\n(14\\n\\nallocations:\\n\\n190.812\\n\\nMiB,\\n\\n0.73%\\n\\ngc\\n\\ntime)\\n\\njulia> norm(b-A*x1)\\n\\n3.8666263141510634e-9\\n\\njulia> @time x2\\n\\n=\\n\\ninv(A)*b;\\n\\n4.550091\\n\\nseconds\\n\\n(21\\n\\nallocations:\\n\\n384.026\\n\\nMiB,\\n\\n1.66%\\n\\ngc\\n\\ntime)\\n\\njulia> norm(b-A*x2)\\n\\n1.682485195063787e-8\\n\\nJulia chooses a suitable algorithm for\\n\\nsolving the equation after checking the\\n\\nproperties of A. For example,\\n\\nit will use back substitution if A is lower triangular.\\n\\nThis explains the result in the following timing experiment.\\n\\njulia> n\\n\\n=\\n\\n5000;\\n\\njulia> b\\n\\n=\\n\\nrandn(n);\\n\\njulia> A\\n\\n=\\n\\ntril(randn(n,n));\\n\\n#\\n\\nrandom\\n\\nlower\\n\\ntriangular\\n\\nmatrix\\n\\n\\x0c\",\n",
       " '11.3\\n\\nSolving linear equations\\n\\n89\\n\\njulia>\\n\\n@time x\\n\\n=\\n\\nA\\\\b;\\n\\n0.042580 seconds\\n\\n(7\\n\\nallocations:\\n\\n39.313\\n\\nKiB)\\n\\njulia>\\n\\nA\\n\\n= randn(n,n);\\n\\n#\\n\\nrandom\\n\\nsquare\\n\\nmatrix\\n\\njulia>\\n\\n@time x\\n\\n=\\n\\nA\\\\b;\\n\\n1.289663 seconds\\n\\n(14\\n\\nallocations:\\n\\n190.812\\n\\nMiB,\\n\\n0.91%\\n\\ngc\\n\\ntime)\\n\\njulia>\\n\\nn\\n\\n= 10000;\\n\\njulia>\\n\\nb\\n\\n= randn(n);\\n\\njulia>\\n\\nA\\n\\n= tril(randn(n,n));\\n\\n#\\n\\nrandom\\n\\nlower\\n\\ntriangular\\n\\nmatrix\\n\\njulia>\\n\\n@time x\\n\\n=\\n\\nA\\\\b;\\n\\n0.157043 seconds\\n\\n(7\\n\\nallocations:\\n\\n78.375\\n\\nKiB)\\n\\njulia>\\n\\nA\\n\\n= randn(n,n);\\n\\n#\\n\\nrandom\\n\\nsquare\\n\\nmatrix\\n\\njulia>\\n\\n@time x\\n\\n=\\n\\nA\\\\b;\\n\\n9.008604 seconds\\n\\n(14\\n\\nallocations:\\n\\n763.093\\n\\nMiB,\\n\\n0.41%\\n\\ngc\\n\\ntime)\\n\\nWhen we double the size from n = 5000 to n = 10000,\\n\\nthe solution time for the\\n\\ntriangular equation increases from 0.04 seconds to 1.29 seconds. This is a factor of\\nabout four, consistent with the n2 complexity of backsubstitution. For the general\\n\\nsquare system, the solution times increases from 0.16 seconds to 9.01 seconds,\\na factor of roughly eight, as we would expect given the order n3 complexity.\\n\\ni.e.,\\n\\nFactor-solve methods for multiple right-hand sides.\\n\\nA linear equation is solved\\n\\nby ﬁrst factorizing A and then solving several simpler equations with the factors\\n\\nof A. This is referred to as a factor-solve scheme. An important application is the\\n\\nsolution of multiple linear equations with the same coeﬃcient matrix and diﬀerent\\n\\nright-hand sides.\\n\\njulia>\\n\\nn\\n\\n= 5000;\\n\\njulia>\\n\\nA\\n\\n= randn(n,n);\\n\\nB\\n\\n=\\n\\nrandn(n,2);\\n\\njulia>\\n\\n#\\n\\nSolve\\n\\nwith\\n\\nright-hand\\n\\nside\\n\\nB[:,1]\\n\\njulia>\\n\\n\\x0c',\n",
       " '90\\n\\n11.4\\n\\nExamples\\n\\n11\\n\\nMatrix inverses\\n\\nPolynomial\\n\\ninterpolation. We\\n\\ncompute\\n\\nthe\\n\\ninterpolating polynomials\\n\\nin Fig-\\n\\nure 11.1 of VMLS. The following code uses the functions vandermonde and linspace\\n\\nfrom the VMLS package.\\n\\n(Th function linspace(a,b,n) returns a vector with n\\n\\nequally spaced numbers in the interval\\n\\n[a, b].) The result is shown in Figure 11.1.\\n\\njulia> t\\n\\njulia> A\\n4×4\\n\\n=\\n\\n=\\n\\n[ -1.1,\\n\\n-0.4,\\n\\n0.2,\\n\\n0.8\\n\\n];\\n\\nvandermonde(t,\\n\\n4)\\n\\nArray{Float64,2}:\\n\\n1.0\\n\\n1.0\\n\\n1.0\\n\\n1.0\\n\\n-1.1\\n\\n-0.4\\n\\n0.2\\n\\n0.8\\n\\n1.21\\n\\n0.16\\n\\n0.04\\n\\n0.64\\n\\n-1.331\\n\\n-0.064\\n\\n0.008\\n\\n0.512\\n\\njulia> b1\\n\\n= [\\n\\n-1.0,\\n\\n1.3,\\n\\n1.0,\\n\\n0.5\\n\\n];\\n\\njulia>\\n\\n\\x0c',\n",
       " '11.5\\n\\nPseudo-inverse\\n\\n91\\n\\nFigure 11.1 Cubic interpolants through two sets of points, shown as circles\\n\\nand squares.\\n\\n11.5\\n\\nPseudo-inverse\\n\\nIn Julia, the pseudo-inverse of a matrix A is obtained with pinv(A). We compute\\n\\nthe pseudo-inverse for the example on page 216 of VMLS using the pinv function,\\nand via the formula A† = R−1QT , where A = QR is the QR factorization of A.\\n\\njulia>\\nA\\n= [-3\\n-4;\\n3×2 Array{Int64,2}:\\n\\n4\\n\\n6;\\n\\n1\\n\\n1]\\n\\n-3\\n\\n-4\\n\\n4\\n\\n1\\n\\n6\\n\\n1\\n\\njulia>\\npinv(A)\\n2×3 Array{Float64,2}:\\n\\n-1.22222\\n\\n-1.11111\\n\\n1.77778\\n\\n0.777778\\n\\n0.888889\\n\\n-1.22222\\n\\njulia>\\n\\nQ, R\\n\\n= qr(A);\\n\\njulia>\\nQ\\n= Matrix(Q)\\n3×2 Array{Float64,2}:\\n\\n-0.588348\\n\\n-0.457604\\n\\n0.784465\\n\\n-0.522976\\n\\n0.196116\\n\\n0.719092\\n\\njulia> R\\n2×2\\n\\nArray{Float64,2}:\\n\\n5.09902\\n\\n7.2563\\n\\n0.0\\n\\n-0.588348\\n\\n-1.0-0.50.00.51.0-2-1012\\x0c',\n",
       " '92\\n\\n11\\n\\nMatrix inverses\\n\\njulia>\\n\\n\\x0c',\n",
       " 'Chapter 12\\n\\nLeast squares\\n\\n12.1\\n\\nLeast squares problem\\n\\nWe take the small\\nleast squares problem of Figure 12.1 in VMLS and check that\\n(cid:107)A ˆx − b(cid:107) is less than (cid:107)Ax − b(cid:107) for some other value of x.\\n\\njulia>\\nA\\n= [ 2\\n0\\n;\\n3×2 Array{Int64,2}:\\n\\n-1\\n\\n1\\n\\n;\\n\\n0\\n\\n2\\n\\n]\\n\\n2\\n\\n-1\\n\\n0\\n\\n0\\n\\n1\\n\\n2\\n\\njulia>\\n\\nb\\n\\n= [ 1,\\n\\n0,\\n\\n-1\\n\\n]\\n\\n3-element Array{Int64,1}:\\n\\n1\\n\\n0\\n\\n-1\\n\\njulia>\\n\\nxhat\\n\\n= [\\n\\n1/3,\\n\\n-1/3\\n\\n]\\n\\n2-element Array{Float64,1}:\\n\\n0.333333\\n\\n-0.333333\\n\\njulia>\\n\\nrhat\\n\\n= A*xhat\\n\\n-b\\n\\n3-element Array{Float64,1}:\\n\\n-0.333333\\n\\n-0.666667\\n\\n0.333333\\n\\njulia>\\n\\nnorm(rhat)\\n\\n0.816496580927726\\n\\njulia>\\n\\nx\\n\\n= [ 1/2,\\n\\n-1/2\\n\\n]\\n\\n2-element Array{Float64,1}:\\n\\n0.5\\n\\n\\x0c',\n",
       " \"94\\n\\n12\\n\\nLeast squares\\n\\n-0.5\\n\\njulia> r\\n\\n=\\n\\nA*x\\n\\n-b\\n\\n3-element\\n\\nArray{Float64,1}:\\n\\n0.0\\n\\n-1.0\\n\\n0.0\\n\\njulia> norm(r)\\n\\n1.0\\n\\n12.2\\n\\nSolution\\n\\nLeast\\n\\nsquares\\n\\nsolution formula.\\n\\nLet’s\\n\\ncheck the\\n\\nsolution formulas\\n\\n(12.5) and\\n\\n(12.6) in VMLS,\\n\\nˆx = (AT A)−1AT b = A† b\\n\\nfor the small example of Figure 12.1) (where ˆx = (1/3, 1/3)).\\n\\njulia> inv(A'*A)*A'*b\\n\\n2-element\\n\\nArray{Float64,1}:\\n\\n0.333333\\n\\n-0.333333\\n\\njulia> pinv(A)*b\\n\\n2-element\\n\\nArray{Float64,1}:\\n\\n0.333333\\n\\n-0.333333\\n(A'*A)*xhat\\n\\njulia>\\n\\n- A'*b\\n\\n# Check\\n\\nthat\\n\\nnormal\\n\\nequations\\n\\nhold\\n\\n2-element\\n\\nArray{Float64,1}:\\n\\n0.0\\n\\n-8.88178e-16\\n\\nOrthogonality principle.\\n\\nLet’s check the orthogonality principle (12.9),\\n\\nfor\\n\\nthe\\n\\nsame example.\\n\\njulia> z\\n=\\n[-1.1,\\njulia> (A*z)'*rhat\\n\\n2.3];\\n\\n2.220446049250313e-16\\n\\njulia>\\n\\njulia>\\n\\nz = [5.3,\\n-1.2];\\n(A*z)'*rhat\\n\\n-6.661338147750939e-16\\n\\n\\x0c\",\n",
       " \"12.3\\n\\nSolving least squares problems\\n\\n95\\n\\n12.3\\n\\nSolving least squares problems\\n\\nJulia uses\\n\\nthe backslash operator\\n\\nto denote the least\\n\\nsquares approximate solu-\\n\\ntion:\\n\\nxhat\\n\\n= A\\\\b.\\n\\n(The same operator\\n\\nis used to solve square systems of\\n\\nlinear\\n\\nequations, and we will see more uses of\\n\\nit in later chapters.)\\n\\njulia>\\n\\nA\\n\\n= randn(100,20);\\n\\nb\\n\\n=\\n\\nrandn(100);\\n\\njulia>\\n\\nx1 =\\n\\nA\\\\b;\\n#\\nLeast\\nsquares\\ninv(A'*A)*(A'*b);\\n\\nusing\\n\\nbackslash\\n\\noperator\\n\\njulia>\\n\\nx2 =\\n\\n#\\n\\nUsing formula\\n\\njulia>\\n\\nx3\\n\\n= pinv(A)*b;\\n\\n# Using\\n\\npseudo-inverse\\n\\njulia>\\n\\nQ,\\n\\nR =\\n\\nqr(A);\\n\\njulia>\\n\\nQ = Matrix(Q);\\n= R\\\\(Q'*b);\\n\\njulia>\\n\\nx4\\n\\n# Using QR\\n\\nfactorization\\n\\njulia>\\n\\nnorm(x1-x2)\\n\\n4.258136640215341e-16\\n\\njulia>\\n\\nnorm(x2-x3)\\n\\n1.3328728382991758e-15\\n\\njulia>\\n\\nnorm(x3-x4)\\n\\n1.3507615689695538e-15\\n\\nComplexity.\\n\\nThe\\ncomplexity of\\nmatrix A is around 2mn2 ﬂops.\\n\\nsolving the\\n\\nleast\\n\\nsquares problem with m × n\\n\\nLet’s check this\\n\\nin Julia by solving a few least\\n\\nsquares problems of diﬀerent dimensions.\\n\\njulia>\\n\\nm\\n\\n= 2000;\\n\\nn\\n\\n=\\n\\n500;\\n\\njulia>\\n\\nA\\n\\n= randn(m,n);\\n\\nb\\n\\n=\\n\\nrandn(m);\\n\\njulia>\\n\\n@time x\\n\\n=\\n\\nA\\\\b;\\n\\n0.190497 seconds\\n\\n(4.07\\n\\nk\\n\\nallocations:\\n\\n12.031\\n\\nMiB, 28.89%\\n\\ngc\\n\\ntime)\\n\\njulia>\\n\\n@time x\\n\\n=\\n\\nA\\\\b;\\n\\n0.120246 seconds\\n\\n(4.07\\n\\nk\\n\\nallocations:\\n\\n12.031\\n\\nMiB)\\n\\njulia>\\n\\nm\\n\\n= 4000;\\n\\nn\\n\\n=\\n\\n500;\\n\\njulia>\\n\\nA\\n\\n= randn(m,n);\\n\\nb\\n\\n=\\n\\nrandn(m);\\n\\njulia>\\n\\n@time x\\n\\n=\\n\\nA\\\\b;\\n\\n0.248510 seconds\\n\\n(4.07\\n\\nk\\n\\nallocations:\\n\\n19.675\\n\\nMiB)\\n\\njulia>\\n\\nm\\n\\n= 2000;\\n\\nn\\n\\n=\\n\\n1000;\\n\\njulia>\\n\\nA\\n\\n= randn(m,n);\\n\\nb\\n\\n=\\n\\nrandn(m);\\n\\njulia>\\n\\n@time x\\n\\n=\\n\\nA\\\\b;\\n\\n0.418181\\n\\nseconds\\n\\n(8.07\\n\\nk\\n\\nallocations:\\n\\n31.608\\n\\nMiB,\\n\\n1.66%\\n\\ngc\\n\\ntime)\\n\\nWe\\n\\ncan see\\n\\nthat doubling m approximately doubles\\n\\nthe\\n\\ncomputation time, and\\n\\ndoubling n increases it by around a factor of\\n\\nfour. The times above can be used to\\n\\nguess the speed of the computer on which it was carried out. For example, using\\nthe last problem solved, the number of ﬂops is around 2mn2 = 4 · 109 , and it took\\n\\naround 0.4 seconds. This suggests a speed of around 10 Gﬂop/sec.\\n\\n\\x0c\",\n",
       " '96\\n\\n12\\n\\nLeast squares\\n\\nMatrix least squares.\\n\\nLet’s solve multiple least squares problems with the same\\n\\nmatrix A and diﬀerent vectors b.\\n\\njulia> A\\n\\njulia> X\\n\\n=\\n\\n=\\n\\nrandn(1000,100);\\n\\nB\\n\\n=\\n\\nrandn(1000,10);\\n\\nA\\\\B;\\n\\njulia> #\\n\\nCheck\\n\\nthat\\n\\nthird\\n\\ncolumn\\n\\nof\\n\\nX\\n\\nis\\n\\nleast\\n\\nsquares\\n\\nsolution\\n\\njulia> #\\n\\nwith\\n\\nthird\\n\\ncolumn\\n\\nof\\n\\nB\\n\\njulia> x3\\n\\n= A\\\\B[:,3];\\n\\njulia> norm(X[:,3]-x3)\\n\\n1.1037090583270415e-16\\n\\n12.4\\n\\nExamples\\n\\nAdvertising purchases. We work out the solution of the optimal advertising pur-\\n\\nchase problem on page 234 of VMLS.\\n\\njulia> R\\n\\n=\\n\\n[ 0.97\\n\\n1.23\\n\\n0.80\\n\\n1.29\\n\\n1.10\\n\\n0.67\\n\\n0.87\\n\\n1.10\\n\\n1.92\\n\\n1.29\\n\\n1.86\\n\\n2.18\\n\\n1.24\\n\\n0.98\\n\\n1.23\\n\\n0.34\\n\\n0.26\\n\\n0.16\\n\\n0.22\\n\\n0.12\\n\\n0.41;\\n\\n0.53;\\n\\n0.62;\\n\\n0.51;\\n\\n0.69;\\n\\n0.54;\\n\\n0.62;\\n\\n0.48;\\n\\n0.71;\\n\\n0.62];\\n\\njulia> m,\\n\\nn =\\n\\nsize(R);\\n\\njulia> vdes =\\n\\n1e3\\n\\n*\\n\\nones(m);\\n\\njulia>\\n\\n\\x0c',\n",
       " '12.4\\n\\nExamples\\n\\n97\\n\\njulia>\\n\\nn\\n\\n= 10;\\n\\n#\\n\\nnumber\\n\\nof\\n\\nlamps\\n\\njulia>\\n\\nlamps =\\n\\n[\\n\\n#\\n\\nx,\\n\\ny\\n\\npositions\\n\\nof\\n\\nlamps\\n\\nand\\n\\nheight\\n\\nabove\\n\\nfloor\\n\\n4.1\\n\\n20.4\\n\\n4;\\n\\n14.1\\n\\n21.3\\n\\n3.5;\\n\\n22.6\\n\\n17.1\\n\\n6;\\n\\n5.5\\n\\n12.3\\n\\n4.0;\\n\\n12.2\\n\\n9.7\\n\\n4.0;\\n\\n15.3\\n\\n13.8\\n\\n6;\\n\\n21.3\\n\\n10.5\\n\\n5.5;\\n\\n3.9\\n\\n3.3\\n\\n5.0;\\n\\n13.1\\n\\n4.3\\n\\n5.0;\\n\\n20.3\\n\\n4.2\\n\\n4.5\\n\\n];\\n\\njulia> N\\n\\n=\\n\\n25;\\n\\n#\\n\\ngrid\\n\\nsize\\n\\njulia> m\\n\\n=\\n\\nN*N;\\n\\n#\\n\\nnumber\\n\\nof\\n\\npixels\\n\\njulia> #\\n\\nconstruct\\n\\nm\\n\\nx\\n\\n2\\n\\nmatrix\\n\\nwith\\n\\ncoordinates of\\n\\npixel\\n\\ncenters\\n\\njulia> pixels\\n\\n=\\n\\nhcat(\\n\\nreshape(\\n\\ncollect(0.5:\\n\\n1\\n\\n:\\n\\nN)\\n\\n*\\n\\nones(1,N),\\n\\nm,\\n\\n1),\\n\\nreshape(\\n\\nones(N,1)\\n\\n*\\n\\ncollect(0.5:\\n\\n1\\n\\n:\\n\\nN)’,\\n\\nm,\\n\\n1) );\\n\\njulia> #\\n\\nThe m\\n\\nx\\n\\nn\\n\\nmatrix\\n\\nA\\n\\nmaps\\n\\nlamp\\n\\npowers\\n\\nto\\n\\npixel\\n\\nintensities.\\n\\njulia> #\\n\\nA[i,j]\\n\\nis\\n\\ninversely\\n\\nproportional\\n\\nto\\n\\nthe squared\\n\\ndistance\\n\\nof\\n\\njulia> #\\n\\nlamp\\n\\nj\\n\\nto\\n\\npixel\\n\\ni.\\n\\njulia> A\\n\\n=\\n\\nzeros(m,n);\\n\\njulia> for\\n\\ni=1:m\\n\\nfor j=1:n\\n\\nA[i,j] =\\n\\n1.0 /\\n\\nnorm([pixels[i,:];\\n\\n0]\\n\\n-\\n\\nlamps[j,:])^2;\\n\\nend;\\n\\nend;\\n\\njulia> A\\n\\n=\\n\\n(m/sum(A))\\n\\n*\\n\\nA;\\n\\n#\\n\\nscale\\n\\nelements\\n\\nof\\n\\nA\\n\\njulia> #\\n\\nLeast\\n\\nsquares\\n\\nsolution\\n\\njulia>\\n\\n\\x0c',\n",
       " '98\\n\\n12\\n\\nLeast squares\\n\\nFigure 12.1 Histogram of pixel\\n\\nillumination values using p = 1 (top) and ˆp\\n\\n(bottom). The target intensity value is one.\\n\\n0.500.751.001.251.500204060801001200.500.751.001.251.50020406080100120\\x0c',\n",
       " 'Chapter 13\\n\\nLeast squares data ﬁtting\\n\\n13.1\\n\\nLeast squares data ﬁtting\\n\\nStraight-line ﬁt.\\n\\nA straight-line ﬁt\\n\\nto time\\n\\nseries data gives an estimate of a\\n\\ntrend line.\\n\\nIn Figure 13.3 of VMLS we apply this\\n\\nto a time series of petroleum\\n\\nconsumption. The ﬁgure is reproduced here as Figure 13.1.\\n\\njulia>\\n\\n#\\n\\nPetroleum\\n\\nconsumption\\n\\nin\\n\\nthousand\\n\\nbarrels/day\\n\\njulia>\\n\\nconsumption\\n\\n=\\n\\npetroleum_consumption_data()\\n\\njulia>\\n\\njulia>\\n\\nn\\n\\nA\\n\\njulia>\\n\\n= length(consumption);\\n\\n= [ ones(n)\\n\\n1:n\\n\\n];\\n\\n\\x0c',\n",
       " '100\\n\\n13\\n\\nLeast squares data ﬁtting\\n\\nFigure 13.1 World petroleum consumption between 1980 and 2013 (dots)\\n\\nand least squares straight-line ﬁt (data from www.eia.gov).\\n\\njulia> scatter(1:m,\\n\\nb,\\n\\nmarkersize\\n\\n=\\n\\n2,\\n\\nlegend\\n\\n=false);\\n\\njulia> plot!(1:m,\\n\\nA*x)\\n\\nThe matrix A in this example has size m × n where m = 15 · 12 = 180 and n = 13.\\n\\nThe ﬁrst column has entries 0, 1, 2, . . . , 179. The remaining columns are formed\\nby vertical stacking of 15 identity matrices of size 12 × 12. The Julia expression\\n\\nvcat([eye(12)\\n\\nfor\\n\\ni=1:15]...) creates an array of 15 identity matrices, and\\n\\nthen stacks them vertically. The plot produced by the code is shown in Figure 13.2.\\n\\nPolynomial ﬁt. We now discuss\\n\\nthe polynomial ﬁtting problem on page 255 in\\n\\nVMLS and the results shown in Figure 13.6. We ﬁrst generate a training set of 100\\n\\npoints and plot them (Figure 13.3).\\n\\njulia> #\\n\\nGenerate\\n\\ntraining\\n\\ndata\\n\\nin\\n\\nthe\\n\\ninterval\\n\\n[-1,\\n\\n1].\\n\\njulia> m\\n\\njulia> t\\n\\njulia> y\\n\\n=\\n\\n=\\n\\n=\\n\\n100;\\n\\n-1\\n\\n.+\\n\\n2*rand(m,1);\\n\\nt.^3\\n\\n-\\n\\nt\\n\\n+\\n\\n0.4\\n\\n./\\n\\n(1\\n\\n.+\\n\\n25*t.^2)\\n\\n+\\n\\n0.10*randn(m,1);\\n\\njulia> using Plots\\n\\njulia> scatter(t,y,legend=false)\\n\\nNext we deﬁne a function that ﬁts the polynomial coeﬃcients using least squares.\\n\\nWe apply the function to ﬁt polynomials of degree 2, 6, 10, 15 to our training set.\\n\\njulia>\\n\\n19801990200020106×1047×1048×1049×104\\x0c',\n",
       " '13.1\\n\\nLeast squares data ﬁtting\\n\\n101\\n\\nFigure 13.2 The dots show vehicle miles traveled in the US, per month,\\n\\nin\\n\\nthe period January 2000–December 2014. The line shows the least squares\\n\\nﬁt of a linear trend and a seasonal component with a 12-month period.\\n\\nFigure 13.3 Training set used in the polynomial ﬁtting example.\\n\\n0501001502.0×1052.2×1052.4×1052.6×105-1.0-0.50.00.51.0-0.50-0.250.000.250.50\\x0c',\n",
       " '102\\n\\n13\\n\\nLeast squares data ﬁtting\\n\\njulia> theta6\\n\\n=\\n\\npolyfit(t,y,7)\\n\\njulia> theta10\\n\\njulia> theta15\\n\\n=\\n\\n=\\n\\npolyfit(t,y,11)\\n\\npolyfit(t,y,16)\\n\\nFinally, we plot\\n\\nthe four polynomials. To simplify this, we ﬁrst write a function\\n\\nthat evaluates a polynomial at all points speciﬁed in a vector x. The plots are in\\n\\nFigure 13.4.\\n\\njulia> polyeval(theta,\\n\\nx)\\n\\n=\\n\\nvandermonde(x,length(theta))*theta;\\n\\njulia> t_plot\\n\\n=\\n\\nlinspace(-1,1,1000);\\n\\njulia> using Plots\\n\\njulia> p\\n\\n=\\n\\nplot(layout=4,\\n\\nlegend=false,\\n\\nylim=(-0.7.\\n\\n0.7))\\n\\njulia> scatter!(t,\\n\\ny,\\n\\nsubplot=1,\\n\\nmarkersize\\n\\n=\\n\\n2)\\n\\njulia> plot!(t_plot,\\n\\npolyeval(theta2,t_plot),\\n\\nsubplot=1)\\n\\njulia> scatter!(t,\\n\\ny,\\n\\nsubplot=2,\\n\\nmarkersize\\n\\n=\\n\\n2)\\n\\njulia> plot!(t_plot,\\n\\npolyeval(theta6,t_plot),\\n\\nsubplot=2)\\n\\njulia> scatter!(t,\\n\\ny,\\n\\nsubplot=3,markersize\\n\\n=\\n\\n2)\\n\\njulia> plot!(t_plot,\\n\\npolyeval(theta10,t_plot),\\n\\nsubplot=3)\\n\\njulia> scatter!(t,\\n\\ny,\\n\\nsubplot=4,\\n\\nmarkersize\\n\\n=\\n\\n2)\\n\\njulia> plot!(t_plot,\\n\\npolyeval(theta15,t_plot),\\n\\nsubplot=4)\\n\\nPiecewise-linear ﬁt.\\n\\nIn the following code least squares is used to ﬁt a piecewise-\\n\\nlinear function to 100 points.\\n\\nIt produces Figure 13.5, which is similar to Figure 13.8\\n\\nin VMLS.\\n\\njulia> #\\n\\ngenerate\\n\\nrandom\\n\\ndata\\n\\njulia> m\\n\\njulia> x\\n\\njulia> y\\n\\n=\\n\\n=\\n\\n=\\n\\n100;\\n\\n-2\\n\\n.+\\n\\n4*rand(m,1);\\n\\n1 .+\\n\\n2*(x.-1)\\n\\n-\\n\\n3*max.(x.+1,0)\\n\\n+\\n\\n4*max.(x.-1,0)\\n\\n+\\n\\n0.3*randn(m,1);\\n\\njulia> #\\n\\nleast\\n\\nsquares\\n\\nfitting\\n\\njulia>\\n\\n\\x0c',\n",
       " '13.1\\n\\nLeast squares data ﬁtting\\n\\n103\\n\\nFigure 13.4 Least squares polynomial ﬁts of degree 1, 6, 10, and 15 to 100\\n\\npoints\\n\\nFigure 13.5 Piecewise-linear ﬁt to 100 points.\\n\\n;\\n\\n-1.0-0.50.00.51.0-0.50-0.250.000.250.50-1.0-0.50.00.51.0-0.50-0.250.000.250.50-1.0-0.50.00.51.0-0.50-0.250.000.250.50-1.0-0.50.00.51.0-0.50-0.250.000.250.50-2-1012-5-4-3-2\\x0c',\n",
       " '104\\n\\n13\\n\\nLeast squares data ﬁtting\\n\\nHouse price regression. We calculate the simple regression model\\n\\nfor predicting\\n\\nhouse sales price from area and number of bedrooms, using the data of 774 house\\n\\nsales in Sacramento.\\n\\njulia> D\\n\\n=\\n\\nhouse_sales_data();\\n\\n#\\n\\ncreates\\n\\n3\\n\\nvectors:\\n\\narea,\\n\\nbeds,\\n\\nprice\\n\\njulia> area =\\n\\nD[\"area\"];\\n\\njulia> beds =\\n\\nD[\"beds\"];\\n\\njulia> price =\\n\\nD[\"price\"];\\n\\njulia> m\\n\\njulia> A\\n\\n=\\n\\n=\\n\\njulia>\\n\\nlength(price);\\n\\n[ ones(m)\\n\\narea\\n\\nbeds\\n\\n];\\n\\n\\x0c',\n",
       " '13.2\\n\\nValidation\\n\\n105\\n\\nFigure 13.6 Hourly temperature at Los Angeles\\n\\nInternational Airport be-\\n\\ntween 12:53AM on May 1, 2016, and 11:53PM on May 5, 2016,\\n\\nshown as\\n\\ncircles. The solid line is the prediction of an auto-regressive model with eight\\n\\ncoeﬃcients.\\n\\njulia>\\n\\nrms(ypred\\n\\n-\\n\\ny)\\n\\n1.0129632612687514\\n\\njulia>\\n\\n#\\n\\nPlot first\\n\\nfive\\n\\ndays\\n\\njulia>\\n\\nusing Plots\\n\\njulia>\\n\\nNplot =\\n\\n24*5\\n\\njulia>\\n\\nscatter(1:Nplot,\\n\\nt[1:Nplot],\\n\\nlegend\\n\\n=false)\\n\\njulia>\\n\\nplot!(M+1:Nplot,\\n\\nypred[1:Nplot-M])\\n\\n13.2\\n\\nValidation\\n\\nPolynomial approximation. We return to the polynomial ﬁtting example of page 100.\\n\\nWe continue with the data vectors t and y in the code on page 100 as the training\\n\\nset, and generate a test set of 100 randomly chosen points generated by the same\\n\\nmethod as used for the training set. We then ﬁt polynomials of degree 0,. . . , 20\\n\\n(i.e., with p = 1, . . . , 21 coeﬃcients) and compute the RMS errors on the training\\n\\nset and the test set. This produces a ﬁgure similar to Figure 13.11 in VMLS, shown\\n\\nhere as Figure 13.7.\\n\\njulia>\\n\\njulia>\\n\\n#\\n\\nm\\n\\nGenerate\\n\\nthe\\n\\ntest\\n\\nset.\\n\\n= 100;\\n\\njulia>\\n\\nt_test =\\n\\n-1\\n\\n.+\\n\\n2*rand(m,1);\\n\\n025507510057.560.062.565.067.5\\x0c',\n",
       " '106\\n\\n13\\n\\nLeast squares data ﬁtting\\n\\nFigure 13.7 RMS error versus polynomial degree for the ﬁtting example in\\n\\nﬁgure 13.4.\\n\\n;\\n\\njulia> y_test\\n\\n=\\n\\nt_test.^3\\n\\n-\\n\\nt_test\\n\\n+\\n\\n0.4\\n\\n./\\n\\n(1\\n\\n.+ 25*t_test.^2)\\n\\n+\\n\\n0.10*randn(m,1);\\n\\njulia> error_train\\n\\n=\\n\\nzeros(21);\\n\\njulia> error_test\\n\\n=\\n\\nzeros(21);\\n\\njulia> for\\n\\np =\\n\\n1:21\\n\\nA\\n\\n= vandermonde(t,p)\\n\\n051015200.30.40.50.60.70.80.9DegreeRelative RMS errorTrainTest\\x0c',\n",
       " '13.2\\n\\nValidation\\n\\n107\\n\\n1, . . . , 774.\\n\\n(We do this by calling the randperm function in the Random package.)\\n\\nWe choose the data points indexed by the ﬁrst 155 elements in the permuted list as\\n\\nfold 1, the next 155 as fold 2, et cetera. The output of the following code outputs\\n\\nis similar to Table 13.1 in VMLS (with diﬀerent numbers because of the random\\n\\nchoice of\\n\\nfolds).\\n\\njulia>\\n\\nD\\n\\n= house_sales_data();\\n\\njulia>\\n\\nprice =\\n\\nD[\"price\"];\\n\\narea\\n\\n=\\n\\nD[\"area\"];\\n\\nbeds\\n\\n=\\n\\nD[\"beds\"];\\n\\njulia>\\n\\njulia>\\n\\nN\\n\\nX\\n\\n= length(price);\\n\\n= [ ones(N)\\n\\narea\\n\\nbeds\\n\\n];\\n\\njulia>\\n\\nnfold =\\n\\ndiv(N,5);\\n\\n#\\n\\nsize\\n\\nof\\n\\nfirst\\n\\nfour\\n\\nfolds\\n\\njulia>\\n\\nimport Random\\n\\njulia>\\n\\nI\\n\\n= Random.randperm(N);\\n\\n#\\n\\nrandom\\n\\npermutation\\n\\nof\\n\\nnumbers\\n\\n1...N\\n\\njulia>\\n\\ncoeff =\\n\\nzeros(5,3);\\n\\nerrors\\n\\n=\\n\\nzeros(5,2);\\n\\njulia>\\n\\nfor k =\\n\\n1:5\\n\\nif k\\n\\n== 1\\n\\nItrain =\\n\\nI[nfold+1:end];\\n\\nItest\\n\\n= I[1:nfold];\\n\\nelseif k\\n\\n== 5\\n\\nItrain =\\n\\nI[1:4*nfold];\\n\\nItest\\n\\n= I[4*nfold+1:end];\\n\\nelse\\n\\nItrain =\\n\\nI[ [1:(k-1)*nfold\\n\\n;\\n\\nk*nfold+1\\n\\n:\\n\\nN]]\\n\\nItest\\n\\n= I[\\n\\n[(k-1)*nfold+1\\n\\n;\\n\\nk*nfold\\n\\n]];\\n\\nend;\\n\\nNtrain =\\n\\nlength(Itrain)\\n\\nNtest\\n\\n= length(Itest)\\n\\n\\x0c',\n",
       " '108\\n\\n13\\n\\nLeast squares data ﬁtting\\n\\n76.194\\n\\n94.3307\\n\\n73.3722\\n\\n27.6299\\n\\n72.9218\\n\\n82.1444\\n\\nValidating time series predictions.\\n\\nIn the next example, we return to the AR\\n\\nmodel of hourly temperatures at LAX. We divide the time series in a training set\\n\\nof 24 days and a test set of 7 days. We ﬁt the AR model to the training set and\\n\\ncalculate\\n\\nthe RMS prediction errors on the\\n\\ntraining and test\\n\\nsets.\\n\\nFigure 13.8\\n\\nshows the model predictions and the the data for the ﬁrst ﬁve days of the test set.\\n\\njulia> t\\n\\njulia> N\\n\\n=\\n\\n=\\n\\ntemperature_data();\\n\\nlength(t);\\n\\njulia> #\\n\\nuse first\\n\\n24\\n\\ndays\\n\\nas\\n\\ntraining\\n\\nset\\n\\njulia> Ntrain\\n\\n=\\n\\n24\\n\\n*\\n\\n24;\\n\\nt_train\\n\\n=\\n\\nt[1:Ntrain];\\n\\njulia> #\\n\\nuse the\\n\\nrest\\n\\nas\\n\\ntest\\n\\nset\\n\\njulia> Ntest =\\n\\nN-Ntrain;\\n\\nt_test\\n\\n=\\n\\nt[Ntrain+1:end];\\n\\njulia> #\\n\\nLeast\\n\\nsquares\\n\\nfit\\n\\nof\\n\\nAR\\n\\npredictor\\n\\nwith\\n\\nmemory\\n\\n8\\n\\njulia> m\\n\\njulia> y\\n\\njulia> A\\n\\n=\\n\\n=\\n\\n=\\n\\njulia>\\n\\nNtrain\\n\\n-\\n\\nM;\\n\\nt_train[M+1:M+m];\\n\\nhcat(\\n\\n[t_train[i:i+m-1]\\n\\nfor\\n\\ni=M:-1:1]...);\\n\\n\\x0c',\n",
       " '13.3\\n\\nFeature engineering\\n\\n109\\n\\nFigure 13.8 Hourly temperature at Los Angeles\\n\\nInternational Airport be-\\n\\ntween 12:53AM on May 25, 2016, and 11:53PM on May 29, 2016,\\n\\nshown\\n\\nas circles. The solid line is the prediction of an auto-regressive model with\\n\\neight coeﬃcients, developed using training data from May 1 to May 24.\\n\\n• area: house area in 1000 square feet,\\n\\n• beds: number of bedrooms,\\n\\n• condo: 1 if a condo, 0 otherwise,\\n\\n• location:\\n\\nof VMLS.\\n\\na number from 1–4,\\n\\nfor the four sets of ZIP codes in Table 13.4\\n\\nThe code computes\\n\\nthe model and makes a scatter plot of actual and predicted\\n\\nprices\\n\\n(Figure 13.9).\\n\\nNote\\n\\nthat\\n\\nthe\\n\\nlast\\n\\nthree\\n\\ncolumns of\\n\\nthe matrix X contain\\n\\nBoolean variables\\n\\n(true or false). We rely on the fact\\n\\nthat Julia treats\\n\\nthis as\\n\\nintegers 1 and 0.\\n\\njulia>\\n\\nD\\n\\n= house_sales_data();\\n\\njulia>\\n\\nprice =\\n\\nD[\"price\"];\\n\\njulia>\\n\\narea\\n\\n= D[\"area\"];\\n\\njulia>\\n\\nbeds\\n\\n= D[\"beds\"];\\n\\njulia>\\n\\ncondo =\\n\\nD[\"condo\"];\\n\\njulia>\\n\\nlocation\\n\\n=\\n\\nD[\"location\"];\\n\\njulia>\\n\\njulia>\\n\\nN\\n\\nX\\n\\njulia>\\n\\n= length(price);\\n\\n= hcat(ones(N),\\n\\narea,\\n\\nmax.(area.-1.5,\\n\\n0),\\n\\nbeds,\\n\\ncondo,\\n\\nlocation\\n\\n.== 2,\\n\\nlocation\\n\\n.==\\n\\n3,\\n\\nlocation\\n\\n.==\\n\\n4\\n\\n);\\n\\n025507510057.560.062.565.067.5\\x0c',\n",
       " '110\\n\\n13\\n\\nLeast squares data ﬁtting\\n\\nFigure 13.9 Scatter plot of actual and predicted prices for a model with eight\\n\\nparameters.\\n\\n175.413\\n\\n-42.7478\\n\\n-17.8784\\n\\n-19.0447\\n\\n-100.911\\n\\n-108.791\\n\\n-24.7652\\n\\njulia>\\n\\nrms(X*theta\\n\\n-\\n\\nprice)\\n\\n#\\n\\nRMS\\n\\nprediction\\n\\nerror\\n\\n68.34428699036884\\n\\njulia>\\n\\nusing\\n\\nPlots\\n\\njulia>\\n\\nscatter(price,\\n\\nX*theta,\\n\\nlims\\n\\n=\\n\\n(0,800));\\n\\njulia>\\n\\nplot!([0,\\n\\n800],\\n\\n[0,\\n\\n800],\\n\\nlinestyle\\n\\n=\\n\\n:dash,\\n\\nlegend\\n\\n=\\n\\nfalse);\\n\\njulia>\\n\\nplot!(xlims\\n\\n=\\n\\n(0,800),\\n\\nylims\\n\\n=\\n\\n(0,800),\\n\\nsize\\n\\n=\\n\\n(500,500));\\n\\njulia>\\n\\nplot!(xlabel\\n\\n=\\n\\n\"Actual\\n\\nprice\",\\n\\nylabel\\n\\n=\\n\\n\"Predicted\\n\\nprice\");\\n\\nWe ﬁnish by a cross-validation of this method. We follow the same approach as for\\n\\nthe simple regression model on page 106, using ﬁve randomly chosen folds. The\\n\\ncode shows\\n\\nthe eight coeﬃcients, and the RMS training and test errors\\n\\nfor each\\n\\nfold.\\n\\n02004006008000200400600800Actual pricePredicted price\\x0c',\n",
       " '13.3\\n\\nFeature engineering\\n\\n111\\n\\njulia>\\n\\nnfold =\\n\\ndiv(N,5);\\n\\njulia>\\n\\nimport Random;\\n\\njulia>\\n\\nI\\n\\n= Random.randperm(N);\\n\\njulia>\\n\\nmodels =\\n\\nzeros(8,5);\\n\\njulia>\\n\\nerrors =\\n\\nzeros(2,5);\\n\\n#\\n\\n#\\n\\njulia>\\n\\nfor k =\\n\\n1:5\\n\\nif k\\n\\n== 1\\n\\nstore\\n\\n8\\n\\ncoefficients\\n\\nfor\\n\\nthe\\n\\n5\\n\\nmodels\\n\\nprediction\\n\\nerrors\\n\\nItrain =\\n\\nI[nfold+1:end];\\n\\nItest\\n\\n= I[1:nfold];\\n\\nelseif k\\n\\n== 5\\n\\nItrain =\\n\\nI[1:4*nfold];\\n\\nItest\\n\\n= I[4*nfold+1:end];\\n\\nelse\\n\\nItrain =\\n\\nI[ [1:(k-1)*nfold\\n\\n;\\n\\nk*nfold+1\\n\\n:\\n\\nN]]\\n\\nItest\\n\\n= I[\\n\\n[(k-1)*nfold+1\\n\\n;\\n\\nk*nfold\\n\\n]];\\n\\nend;\\n\\nNtrain =\\n\\nlength(Itrain)\\n\\nNtest\\n\\n= length(Itest)\\n\\n\\x0c',\n",
       " '112\\n\\n13\\n\\nLeast squares data ﬁtting\\n\\n\\x0c',\n",
       " 'Chapter 14\\n\\nLeast squares classiﬁcation\\n\\n14.1\\n\\nClassiﬁcation\\n\\nBoolean values.\\n\\nJulia has\\n\\nthe Boolean values true and false.\\n\\nThese are au-\\n\\ntomatically converted to the numbers 1 and 0 when they combined in numerical\\n\\nexpressions.\\nIn VMLS we use the encoding (for classiﬁers) where True corresponds\\nto +1 and False corresponds to −1. We can get our encoding from a Julia Boolean\\n\\nvalue b using 2*b-1, or via the ternary conditional operation b\\n\\n?\\n\\n1 :\\n\\n-1.\\n\\njulia>\\n\\ntf2pm1(b)\\n\\n=\\n\\n2*b-1\\n\\njulia>\\n\\nb\\n\\n= true\\n\\ntrue\\n\\njulia>\\n\\ntf2pm1(b)\\n\\n1\\n\\njulia>\\n\\nb\\n\\n= false\\n\\nfalse\\n\\njulia>\\n\\ntf2pm1(b)\\n\\n-1\\n\\njulia>\\n\\nb\\n\\n= [ true,\\n\\nfalse,\\n\\ntrue\\n\\n]\\n\\n3-element Array{Bool,1}:\\n\\ntrue\\n\\nfalse\\n\\ntrue\\n\\njulia>\\n\\ntf2pm1.(b)\\n\\n3-element Array{Int64,1}:\\n\\n1\\n\\n-1\\n\\n1\\n\\n\\x0c',\n",
       " \"114\\n\\n14\\n\\nLeast squares classiﬁcation\\n\\nConfusion matrix.\\n\\nLet’s\\n\\nsee how we would evaluate\\n\\nthe prediction errors and\\n\\nconfusion matrix, given a set of data y and predictions yhat, both stored as arrays\\n\\n(vectors) of Boolean values, of\\n\\nlength N.\\n\\njulia> #\\n\\nCount\\n\\nerrors\\n\\nand\\n\\ncorrect\\n\\npredictions\\n\\njulia> Ntp(y,yhat)\\n\\n=\\n\\nsum(\\n\\n(y\\n\\n.==\\n\\ntrue)\\n\\n.&\\n\\n(yhat\\n\\n.==\\n\\ntrue)\\n\\n);\\n\\njulia> Nfn(y,yhat)\\n\\n=\\n\\nsum(\\n\\n(y\\n\\n.==\\n\\ntrue)\\n\\n.&\\n\\n(yhat\\n\\n.==\\n\\nfalse)\\n\\n);\\n\\njulia> Nfp(y,yhat)\\n\\n=\\n\\nsum(\\n\\n(y\\n\\n.==\\n\\nfalse)\\n\\n.&\\n\\n(yhat .==\\n\\ntrue)\\n\\n);\\n\\njulia> Ntn(y,yhat)\\n\\n=\\n\\nsum(\\n\\n(y\\n\\n.==\\n\\nfalse)\\n\\n.&\\n\\n(yhat .==\\n\\nfalse)\\n\\n);\\n\\njulia> error_rate(y,yhat)\\n\\n=\\n\\n(Nfn(y,yhat)\\n\\n+\\n\\nNfp(y,yhat))\\n\\n/\\n\\nlength(y);\\n\\njulia> confusion_matrix(y,yhat)\\n\\n=\\n\\n[\\n\\nNtp(y,yhat)\\n\\nNfn(y,yhat);\\n\\nNfp(y,yhat)\\n\\nNtn(y,yhat)\\n\\n];\\n\\njulia> y\\n\\n=\\n\\nrand(Bool,100);\\n\\nyhat\\n\\n=\\n\\nrand(Bool,100);\\n\\njulia> confusion_matrix(y,yhat)\\n2×2\\n\\nArray{Int64,2}:\\n\\n25\\n\\n23\\n\\n29\\n\\n23\\n\\njulia> error_rate(y,yhat)\\n\\n0.52\\n\\nThe dots that precede == and & cause them to be evaluated elementwise. When\\n\\nwe sum the Boolean vectors,\\n\\nthey are converted to integers.\\n\\nIn the last\\n\\nsection\\n\\nof\\n\\nthe\\n\\ncode we\\n\\ngenerate\\n\\ntwo\\n\\nrandom Boolean vectors,\\n\\nso we\\n\\nexpect\\n\\nthe\\n\\nerror\\n\\nrate\\n\\nto be\\n\\naround 50%.\\n\\nIn the\\n\\ncode\\n\\nabove, we\\n\\ncompute\\n\\nthe\\n\\nerror\\n\\nrate\\n\\nfrom\\n\\nthe numbers of\\n\\nfalse negatives and false positives.\\n\\nA more\\n\\ncompact\\n\\nexpression\\n\\nfor\\n\\nthe error\\n\\nrate is avg(y .!=\\n\\nyhat). The VMLS package contains\\n\\nthe function\\n\\nconfusion_matrix(y,\\n\\nyhat).\\n\\n14.2\\n\\nLeast squares classiﬁer\\n\\nWe can evaluate\\n\\nˆf (x) = sign( ˜f (x)) using ftilde(x)>0, which returns a Boolean\\n\\nvalue.\\n\\njulia> ftilde(x)\\n\\n=\\n\\nx'*beta .+\\n\\nv\\n\\n#\\n\\nRegression\\n\\nmodel\\n\\njulia> fhat(x)\\n\\n=\\n\\nftilde(x)\\n\\n>\\n\\n0\\n\\n#\\n\\nRegression\\n\\nclassifier\\n\\nIris ﬂower\\n\\nclassiﬁcation.\\n\\nThe\\n\\nIris data set\\n\\ncontains of 150 examples of\\n\\nthree\\n\\ntypes of\\n\\niris ﬂowers. There are 50 examples of each class. For each example,\\n\\nfour\\n\\nfeatures are provided. The following code reads\\nin a dictionary containing three\\n50 × 4 matrices setosa, versicolor, virginica with the examples for each class,\\nand then computes a Boolean classiﬁer that distinguishes Iris Virginica from the\\n\\nthe other two classes.\\n\\n\\x0c\",\n",
       " '14.3\\n\\nMulti-class classiﬁers\\n\\n115\\n\\njulia>\\n\\njulia>\\n\\nD\\n\\n#\\n\\n= iris_data();\\n\\nCreate\\n\\n150x4\\n\\ndata\\n\\nmatrix\\n\\njulia>\\n\\niris\\n\\n= vcat(D[\"setosa\"],\\n\\nD[\"versicolor\"],\\n\\nD[\"virginica\"])\\n\\n#\\n\\ny\\n\\nA\\n\\njulia>\\n\\njulia>\\n\\njulia>\\n\\njulia>\\n\\ny[k] is\\n\\ntrue\\n\\n(1)\\n\\nif\\n\\nvirginica,\\n\\nfalse\\n\\n(0)\\n\\notherwise\\n\\n= [ zeros(Bool,\\n\\n50);\\n\\nzeros(Bool,\\n\\n50);\\n\\nones(Bool,\\n\\n50)\\n\\n];\\n\\n= [ ones(150)\\n\\niris\\n\\n]\\n\\n\\x0c',\n",
       " '116\\n\\n14\\n\\nLeast squares classiﬁcation\\n\\njulia> K\\n\\n=\\n\\n4;\\n\\njulia> y\\n\\n=\\n\\nrand(1:K,\\n\\n100);\\n\\nyhat\\n\\n=\\n\\nrand(1:K,\\n\\n100);\\n\\njulia> C\\n4×4\\n\\n=\\n\\nconfusion_matrix(y,\\n\\nyhat,\\n\\nK)\\n\\nArray{Float64,2}:\\n\\n4.0\\n\\n9.0\\n\\n8.0\\n\\n5.0\\n\\n5.0\\n\\n4.0\\n\\n4.0\\n\\n13.0\\n\\n3.0\\n\\n8.0\\n\\n7.0\\n\\n11.0\\n\\n7.0\\n\\n2.0\\n\\n7.0\\n\\n3.0\\n\\njulia> error_rate(y,\\n\\nyhat),\\n\\n1-sum(diag(C))/sum(C)\\n\\n(0.82, 0.8200000000000001)\\n\\nThe function confusion_matrix is included in the VMLS package.\\n\\nLeast squares multi-class classiﬁer.\\n\\nA K -class classiﬁer (with regression model)\\n\\ncan be expressed as\\n\\nˆf (x) = argmax\\n\\nk=1,...,K\\n\\n˜fk (x),\\n\\nwhere\\n\\n˜fk (x) = xT θk . The n-vectors θ1 , . . . , θK are the coeﬃcients or parameters\\nin the model. We can express this in matrix-vector notation as\\n\\nˆf (x) = argmax(xT Θ),\\nthe n × K matrix of model\\n\\nwhere Θ = [\\n\\nθ1\\n\\n· · ·\\n\\nθK ]\\n\\nis\\n\\ncoeﬃcients, and the\\n\\nargmax of a row vector has the obvious meaning.\\n\\nLet’s\\n\\nsee how to express\\n\\nthis\\n\\nin Julia.\\n\\nIn Julia the function argmax(u) ﬁnds\\n\\nthe index of the largest entry in the row or column vector u,\\n\\ni.e., argmaxk uk . To\\nto matrices, we deﬁne a function row_argmax that\\nreturns a vector\\n\\nextend this\\n\\nwith,\\n\\nfor each row, the index of the largest entry in that row.\\n\\njulia> row_argmax(u)\\n\\n=\\n\\n[\\n\\nargmax(u[i,:])\\n\\nfor\\n\\ni\\n\\n=\\n\\n1:size(u,1)\\n\\n]\\n\\njulia> A\\n4×5\\n\\n=\\n\\nrandn(4,5)\\n\\nArray{Float64,2}:\\n\\n1.42552\\n\\n0.766725\\n\\n1.7106\\n\\n-1.08668\\n\\n-0.492051\\n\\n-0.507653\\n\\n-0.158288\\n\\n-1.37703\\n\\n-0.388304\\n\\n0.00290895\\n\\n-1.43499\\n\\n-1.18238\\n\\n-0.182795\\n\\n-0.428589\\n\\n-0.87592\\n\\n-2.18407\\n\\n1.28363\\n\\n0.749702\\n\\n-0.304138\\n\\n0.0165654\\n\\njulia>\\n\\nrow_argmax(A)\\n\\n4-element\\n\\nArray{Int64,1}:\\n\\n3\\n\\n5\\n\\n3\\n\\n2\\n\\nIf a data set with N examples is stored as an n × N data matrix X, and Theta is\\nan n × K matrix with the coeﬃcient vectors θk as its columns, then we can now\\ndeﬁne a function\\n\\n\\x0c',\n",
       " \"14.3\\n\\nMulti-class classiﬁers\\n\\n117\\n\\nfhat(X,Theta) =\\n\\nrow_argmax(X'*Theta)\\n\\nto ﬁnd the N -vector of predictions.\\n\\nMatrix least squares.\\n\\nLet’s use least squares to ﬁnd the coeﬃcient matrix Θ for a\\n\\nmulti-class classiﬁer with n features and K classes,\\nfrom a data set of N examples.\\nthe data is given as an n × N matrix X and an N -vector y cl\\nWe will assume\\nwith entries in {1, . . . , K } that give the classes of the examples. The least squares\\n\\nob jective can be expressed as a matrix norm squared,\\n\\n(cid:107)X T Θ − Y (cid:107)2 ,\\n\\nwhere Y is the N × K vector with\\n\\nYij =\\n\\n(cid:26)\\n\\n1\\n\\n−1\\n\\ny cl\\ny cl\\ni\\n\\ni = j\\n(cid:54)= j.\\n\\nIn other words, the rows of Y describe the classes using one-hot encoding, converted\\nfrom 0/1 to −1/ + 1 values. The least squares solution is given by ˆΘ = (X T )†Y .\\n\\nLet’s see how to express this in Julia.\\n\\njulia>\\n\\nfunction\\n\\none_hot(ycl,K)\\n\\nN\\n\\nY\\n\\n= length(ycl)\\n\\n= zeros(N,K)\\n\\nfor j\\n\\nin 1:K\\n\\nY[findall(ycl\\n\\n.==\\n\\nj),\\n\\nj]\\n\\n.=\\n\\n1\\n\\nend\\n\\nreturn\\n\\nY\\n\\nend;\\n\\njulia> K\\n\\n=\\n\\n4;\\n\\njulia> ycl\\n\\n= rand(1:K,6)\\n\\n6-element\\n\\nArray{Int64,1}:\\n\\n4\\n\\n2\\n\\n3\\n\\n1\\n\\n2\\n\\n1\\n\\njulia> Y\\n\\n=\\n\\none_hot(ycl,\\n\\nK)\\n\\n64 ArrayFloat64,2:\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n1.0\\n\\n0.0\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n1.0\\n\\n0.0\\n\\n1.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n\\x0c\",\n",
       " \"118\\n\\n14\\n\\nLeast squares classiﬁcation\\n\\n0.0\\n\\n1.0\\n\\n1.0\\n\\n0.0\\n\\njulia> 2*Y\\n6×4\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n.-\\n\\n1\\n\\nArray{Float64,2}:\\n\\n-1.0\\n\\n-1.0\\n\\n-1.0\\n\\n1.0\\n\\n-1.0\\n\\n-1.0\\n\\n1.0\\n\\n-1.0\\n\\n-1.0\\n\\n1.0\\n\\n1.0\\n\\n-1.0\\n\\n-1.0\\n\\n-1.0\\n\\n1.0\\n\\n-1.0\\n\\n-1.0\\n\\n-1.0\\n\\n1.0\\n\\n-1.0\\n\\n-1.0\\n\\n-1.0\\n\\n-1.0\\n\\n-1.0\\n\\nUsing the functions we have deﬁned, the matrix least squares multi-class classiﬁer\\n\\ncan be computed in a few lines.\\n\\njulia> function\\n\\nls_multiclass(X,ycl,K)\\n\\nn, N\\n\\n=\\n\\nTheta\\n\\nsize(X)\\n= X'\\n\\n\\x0c\",\n",
       " \"14.3\\n\\nMulti-class classiﬁers\\n\\n119\\n\\njulia>\\n\\nXtest =\\n\\n[\\n\\nsetosa[I1[41:end],:];\\n\\nversicolor[I2[41:end],:]\\n\\nvirginica[I3[41:end],:]\\n\\n]';\\n\\n# 4x30\\n\\ndata\\n\\nmatrix\\n\\njulia>\\n\\nXtest\\n\\njulia>\\n\\nytest\\n\\n=\\n\\n=\\n\\n[\\n\\nones(1,30);\\n\\nXtest\\n\\n];\\n\\n#\\n\\n5x30\\n\\ndata\\n\\nmatrix\\n\\n[ones(10);\\n\\n2*ones(10);\\n\\n3*ones(10)];\\n\\njulia>\\n\\nTheta,\\n\\nyhat\\n\\n=\\n\\nls_multiclass(Xtrain,\\n\\nytrain,\\n\\n3);\\n\\nCtrain\\n\\n=\\n\\nconfusion_matrix(ytrain,\\n\\nyhat,\\n\\n3)\\n\\njulia>\\n3×3\\n\\nArray{Float64,2}:\\n\\n40.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n28.0\\n\\n6.0\\n\\n0.0\\n\\n12.0\\n\\n34.0\\n\\njulia>\\n\\nerror_train\\n\\n=\\n\\nerror_rate(ytrain,\\n\\nyhat)\\n\\n0.15\\n\\njulia>\\n\\nyhat =\\n\\nrow_argmax(Xtest'*theta)\\n\\njulia> Ctest =\\n3×3\\n\\nArray{Float64,2}:\\n\\nconfusion_matrix(ytest,\\n\\nyhat,\\n\\n3)\\n\\n10.0\\n\\n0.0\\n\\n0.0\\n\\n0.0\\n\\n7.0\\n\\n2.0\\n\\n0.0\\n\\n3.0\\n\\n8.0\\n\\njulia> error_test\\n\\n=\\n\\nerror_rate(ytest,\\n\\nyhat)\\n\\n0.16666666666666666\\n\\n\\x0c\",\n",
       " '120\\n\\n14\\n\\nLeast squares classiﬁcation\\n\\n\\x0c',\n",
       " 'Chapter 15\\n\\nMulti-objective least squares\\n\\n15.1\\n\\nMulti-objective least squares\\n\\nLet’s write a functiont that solves the multi-ob jective least squares problem, with\\n\\ngiven positive weights.\\n\\nThe data are a list\\n\\n(or array) of coeﬃcient matrices\\n\\n(of\\n\\npossibly diﬀerent heights) As, a matching list of (right-hand side) vectors bs, and\\n\\nthe weights, given as an array or list, lambdas.\\n\\njulia>\\n\\nfunction\\n\\nmols_solve(As,bs,lambdas)\\n\\nk\\n\\n= length(lambdas);\\n\\nAtil\\n\\nbtil\\n\\n=\\n\\n=\\n\\nvcat([sqrt(lambdas[i])*As[i]\\n\\nfor\\n\\ni=1:k]...)\\n\\nvcat([sqrt(lambdas[i])*bs[i]\\n\\nfor\\n\\ni=1:k]...)\\n\\n\\x0c',\n",
       " '122\\n\\n15\\n\\nMulti-objective least squares\\n\\nFigure 15.1 Weighted-sum least squares solution ˆx(λ) as a function of λ for\\n\\na bi-criterion least squares problem with ﬁve variables.\\n\\nx[:,k] =\\n\\nmols_solve(As,\\n\\nbs,\\n\\n[1,\\n\\nlambdas[k]])\\n\\nJ1[k]\\n\\nJ2[k]\\n\\n=\\n\\n=\\n\\nnorm(As[1]*x[:,k]\\n\\nnorm(As[2]*x[:,k]\\n\\n-\\n\\n-\\n\\nbs[1])^2\\n\\nbs[2])^2\\n\\nend;\\n\\njulia> using Plots\\n\\njulia> #\\n\\nplot\\n\\nsolution\\nversus\\nx\\',\\n\\nxscale\\n\\nlambda\\n\\njulia> plot(lambdas,\\n\\n=\\n\\n:log10,\\n\\nxlabel\\n\\n=\\n\\n\"lambda\");\\n\\njulia>\\n\\nplot!(xlims\\n\\n=\\n\\n(1e-4,1e4));\\n\\njulia>\\n\\n# plot\\n\\ntwo\\n\\nobjectives\\n\\nversus\\n\\nlambda\\n\\njulia>\\n\\nplot(lambdas,\\n\\nJ1,\\n\\nxscale\\n\\njulia>\\n\\nplot!(lambdas,\\n\\nJ2,\\n\\nlabel\\n\\n=\\n\\n=\\n\\n:log10,\\n\\nlabel\\n\\n= \"J_1\");\\n\\n\"J_2\",\\n\\nxlabel\\n\\n= \"lambda\",\\n\\nxlims\\n\\n=\\n\\n(1e-4,1e4));\\n\\njulia>\\n\\n# plot\\n\\ntrade-off\\n\\ncurve\\n\\njulia>\\n\\nplot(J1,\\n\\nJ2,\\n\\nxlabel=\"J1\",\\n\\nylabel\\n\\n=\\n\\n\"J2\",\\n\\nlegend=false);\\n\\njulia>\\n\\n# add\\n\\n(single-objective)\\n\\nend\\n\\npoints\\n\\nto\\n\\ntrade-off\\n\\ncurve\\n\\njulia>\\n\\n10-4 10-2 100 102 104 -0.50-0.250.000.25lambday1y2y3y4y5\\x0c',\n",
       " '15.1\\n\\nMulti-objective least squares\\n\\n123\\n\\nFigure 15.2 Ob jective\\n\\n(cid:107)A2 ˆx(λ)b2 (cid:107)2\\nure 15.1.\\n\\nfunctions J1 = (cid:107)A1 ˆx(λ)b1 (cid:107)2\\nline) and J2 =\\n(red line) as functions of λ for the bi-criterion problem in ﬁg-\\n\\n(blue\\n\\nFigure 15.3 Optimal trade-oﬀ curve for the bi-criterion least squares problem\\n\\nof ﬁgures 15.1 and 15.2.\\n\\n10-4 10-2 100 102 104 5.07.510.012.515.017.5lambdaJ_1J_27.510.012.515.017.55.07.510.012.515.017.5J1J2\\x0c',\n",
       " '124\\n\\n15\\n\\nMulti-objective least squares\\n\\nFigure 15.4 Hourly ozone level at Azusa, California, during the ﬁrst 14 days\\n\\nof July 2014 (California Environmental Protection Agency, Air Re- sources\\n\\nBoard, www.arb.ca.gov). Measurements start at 12AM on July 1st, and end\\n\\nat 11PM on July 14. Note the large number of missing measure- ments.\\n\\nIn\\n\\nparticular, all 4AM measurements are missing.\\n\\n15.2\\n\\nControl\\n\\n15.3\\n\\nEstimation and inversion\\n\\nEstimating a periodic time series. We consider\\n\\nthe example of Figure 15.4 in\\n\\nVMLS. We start by loading the data, as a vector with hourly ozone levels,\\n\\nfor a\\n\\nperiod of 14 days. Missing measurements have a value NaN (for Not a Number).\\n\\nThe plot command skips those values (Figure 15.4).\\n\\njulia> ozone =\\n\\nozone_data();\\n\\n#\\n\\na\\n\\nvector\\n\\nof\\n\\nlength\\n\\n14*24\\n\\n=\\n\\n336\\n\\njulia> k\\n\\n=\\n\\n14;\\n\\nN\\n\\n=\\n\\nk*24;\\n\\njulia> plot(1:N,\\n\\nozone,\\n\\nyscale\\n\\n=\\n\\n:log10,\\n\\nmarker\\n\\n= :circle,\\n\\nlegend=false)\\n\\nNext we use\\n\\nthe mols_solve function to make a periodic ﬁt,\\n\\nfor\\n\\nthe values\\n\\nλ = 1 and λ = 10. The Julia code isnan is used to ﬁnd and discard the missing\\n\\nmeasurements. The results are shown in Figures 15.5 and 15.6.\\n\\njulia> A\\n\\n=\\n\\nvcat(\\n\\n[eye(24)\\n\\nfor\\n\\ni\\n\\n=\\n\\n1:k]...)\\n\\njulia> #\\n\\nperiodic\\n\\ndifference\\n\\nmatrix\\n\\njulia> D\\n\\n=\\n\\n-eye(24)\\n\\njulia> ind\\n\\n= [k\\n\\nfor\\n\\n+\\n\\nk\\n\\n[zeros(23,1)\\n\\neye(23);\\n\\n1\\n\\nzeros(1,23)];\\n\\nin\\n\\n1:length(ozone)\\n\\nif\\n\\n!isnan(ozone[k])];\\n\\njulia> As\\n\\n= [A[ind,:],\\n\\nD]\\n\\n010020030010-2.4 10-2.1 10-1.8 10-1.5 10-1.2 \\x0c',\n",
       " '15.4\\n\\nRegularized data ﬁtting\\n\\n125\\n\\nFigure 15.5 Smooth periodic least squares ﬁt to logarithmically transformed\\n\\nmeasurements, using λ = 1.\\n\\njulia>\\n\\nbs =\\n\\n[log.(ozone[ind]),\\n\\nzeros(24)]\\n\\njulia>\\n\\njulia>\\n\\n#\\n\\nx\\n\\nsolution\\n\\nfor\\n\\nlambda\\n\\n=\\n\\n1\\n\\n= mols_solve(\\n\\nAs,\\n\\nbs,\\n\\n[1,\\n\\n1])\\n\\njulia>\\n\\nscatter(1:N,\\n\\nozone,\\n\\nyscale\\n\\n=\\n\\n:log10,\\n\\nlegend=false)\\n\\njulia>\\n\\nplot!(1:N,\\n\\nvcat([exp.(x)\\n\\nfor\\n\\ni\\n\\n=\\n\\n1:k]...))\\n\\njulia>\\n\\njulia>\\n\\n#\\n\\nx\\n\\nsolution\\n\\nfor\\n\\nlambda\\n\\n=\\n\\n100\\n\\n= mols_solve(\\n\\nAs,\\n\\nbs,\\n\\n[1,\\n\\n100])\\n\\njulia>\\n\\nscatter(1:N,\\n\\nozone,\\n\\nyscale\\n\\n=\\n\\n:log10,\\n\\nlegend=false)\\n\\njulia>\\n\\nplot!(1:N,\\n\\nvcat([exp.(x)\\n\\nfor\\n\\ni\\n\\n=\\n\\n1:k]...))\\n\\n15.4\\n\\nRegularized data ﬁtting\\n\\nExample.\\n\\nNext we consider the small regularized data ﬁtting example of page 329\\n\\nof VMLS. We ﬁt a model\\n\\nˆf (x) =\\n\\nθk fk (x)\\n\\n5(cid:88)\\n\\nk=1\\n\\nwith basis\\n\\nfunctions f1 (x) = 1 and fk+1 (x) = sin(ωk x + φk )\\nN = 20 data points. We use the values of ωk , φk given in the text. We ﬁt\\n(y (i) − 5(cid:88)\\nmodel by solving a sequence of regularized least squares problems with ob jective\\n\\nθk fk (x(i) ))2 + λ\\n\\nN(cid:88)\\n\\n5(cid:88)\\n\\nfor k = 1, . . . , 4 to\\n\\nthe\\n\\nθ2\\nk .\\n\\ni=1\\n\\nk=1\\n\\nk=2\\n\\n010020030010-2.4 10-2.1 10-1.8 10-1.5 10-1.2 \\x0c',\n",
       " '126\\n\\n15\\n\\nMulti-objective least squares\\n\\nFigure 15.6 Smooth periodic least squares ﬁt to logarithmically transformed\\n\\nmeasurements, using λ = 100.\\n\\nThe two plots are shown in Figures 15.7 and 15.8.\\n\\njulia> #\\n\\nImport\\n\\ndata\\n\\nas\\n\\nvectors\\n\\nxtrain,\\n\\nytrain,\\n\\nxtest,\\n\\nytest\\n\\njulia> D\\n\\n=\\n\\nregularized_fit_data();\\n\\njulia> xtrain\\n\\n=\\n\\nD[\"xtrain\"];\\n\\nytrain\\n\\n=\\n\\nD[\"ytrain\"];\\n\\njulia> xtest =\\n\\nD[\"xtest\"];\\n\\nytest\\n\\n=\\n\\nD[\"ytest\"];\\n\\njulia> N\\n\\n=\\n\\nlength(ytrain);\\n\\njulia> Ntest =\\n\\nlength(ytest);\\n\\njulia> p\\n\\n=\\n\\n5;\\n\\njulia> omega =\\n\\n[\\n\\n13.69;\\n\\n3.55;\\n\\n23.25;\\n\\n6.03\\n\\n];\\n\\njulia> phi\\n\\n= [\\n\\n0.21;\\n\\njulia> A\\n\\n=\\n\\nhcat(ones(N),\\n\\n0.02;\\n-1.87;\\n1.72\\n];\\nsin.(xtrain*omega\\'\\n\\nones(N)*phi\\'));\\n\\n+\\n\\njulia> Atest =\\nhcat(ones(Ntest),\\nsin.(xtest*omega\\'\\nones(Ntest)*phi\\'));\\n\\n+\\n\\njulia> npts =\\n\\n100;\\n\\njulia> lambdas\\n\\n=\\n\\n10\\n\\n.^\\n\\nlinspace(-6,6,npts);\\n\\njulia> err_train\\n\\n=\\n\\nzeros(npts);\\n\\njulia> err_test\\n\\n= zeros(npts);\\n\\njulia> thetas\\n\\n=\\n\\nzeros(p,npts);\\n\\njulia> for\\n\\nk =\\n\\n1:npts\\n\\ntheta\\n\\n= mols_solve([\\n\\nA,\\n\\n[zeros(p-1)\\n\\neye(p-1)]],\\n\\n[\\n\\nytrain,\\n\\nzeros(p-1)\\n\\n],\\n\\n[1,\\n\\nlambdas[k]])\\n\\nerr_train[k]\\n\\n=\\n\\nrms(ytrain\\n\\n-\\n\\nA*theta);\\n\\nerr_test[k]\\n\\nthetas[:,k]\\n\\n=\\n\\n=\\n\\nrms(ytest\\n\\n-\\n\\nAtest*theta);\\n\\ntheta;\\n\\n010020030010-2.4 10-2.1 10-1.8 10-1.5 10-1.2 \\x0c',\n",
       " '15.5\\n\\nComplexity\\n\\n127\\n\\nFigure 15.7 RMS training and test errors as a function of the regularization\\n\\nparameter λ.\\n\\nend;\\n\\njulia> using Plots\\n\\njulia> #\\n\\nPlot\\n\\nRMS\\n\\nerrors\\n\\njulia> plot(lambdas,\\n\\nerr_train,\\n\\nxscale\\n\\njulia> plot!(lambdas,\\n\\nerr_test,\\n\\nxscale\\n\\njulia> plot!(xlabel\\n\\n=\\n\\n\"lambda\",\\n\\nylabel\\n\\n=\\n\\n=\\n\\n=\\n\\n:log10, label\\n\\n:log10, label\\n\\n=\\n\\n=\\n\\n\"Train\")\\n\\n\"Test\")\\n\\n\"RMS\\n\\nerror\",\\n\\nxlim\\n\\n= (1e-6,\\n\\n1e6));\\n\\njulia> #\\n\\nPlot\\n\\njulia> plot(lambdas,\\n\\nxscale\\n\\ncoefficients\\nthetas\\',\\n\\njulia>\\n\\nplot!(xlabel\\n\\n=\\n\\n\"lambda\",\\n\\nxlim\\n\\n=\\n\\n=\\n\\n:log10)\\n\\n(1e-6,\\n\\n1e6));\\n\\n15.5\\n\\nComplexity\\n\\nThe kernel trick.\\n\\nLet’s check the kernel trick, described in §15.5.2, to ﬁnd ˆx, the\\n\\nminimizer of\\n\\n(cid:107)Ax − b(cid:107)2 + λ(cid:107)x − xdes (cid:107)2 ,\\nwhere A is an m × n matrix and λ > 0. We’ll compute ˆx two ways. First, the na¨ıve\\n\\nway, and then, using the kernel trick. We use the fact that if\\n\\n(cid:20)\\n\\n(cid:21)\\n\\nAT√\\nλI\\n\\n= QR,\\n\\nthen\\n\\n(AAT + λI )−1 = (RT QT QR)−1 = R−1R−T .\\n\\n10-6 10-4 10-2 100 102 104 106 0.20.40.60.81.0lambdaRMS errorTrainTest\\x0c',\n",
       " '128\\n\\n15\\n\\nMulti-objective least squares\\n\\nFigure 15.8 The regularization path.\\n\\njulia> m\\n\\njulia> A\\n\\n=\\n\\n=\\n\\n100;\\n\\nn\\n\\n=\\n\\n5000;\\n\\nrandn(m,n);\\n\\nb\\n\\n=\\n\\nrandn(m);\\n\\nxdes\\n\\n=\\n\\nrandn(n);\\n\\njulia> lam\\n\\n= 2.0;\\n\\njulia> #\\n\\nFind\\n\\nx\\n\\nthat\\n\\nminimizes\\n\\n||Ax-b||^2\\n\\n+\\n\\nlambda\\n\\n||x||^2\\n\\njulia>\\n\\n10-6 10-4 10-2 100 102 104 106 -1.5-1.0-0.50.00.51.01.5lambday1y2y3y4y5\\x0c',\n",
       " 'Chapter 16\\n\\nConstrained least squares\\n\\n16.1\\n\\nConstrained least squares problem\\n\\nIn the examples in this section, we use the cls_solve function, given later, to ﬁnd\\n\\nthe constrained least squares solution.\\n\\nPiecewise polynomial. We ﬁt a function\\n\\nˆf (x) = q(x)\\n\\nfor x ≤ a and\\nˆf (x) = p(x)\\np(cid:48) (a) = q (cid:48) (a),\\ni.e.,\\nthe\\ntwo polynomials have matching value and slope at\\nthe\\nknot point a. We have data points x1 , . . . , xM ≤ a and xM +1 , . . . , xN > a and\\ncorresponding values y1 , . . . , yN .\\nIn the example we take a = 0, polynomials p and\\nq of degree 3, and N = 2M = 140. The code creates a ﬁgure similar to Figure 16.1\\n\\nto p(a) = q(a) and\\n\\nˆf : R → R to some given data, where\\n\\nfor x > a,\\n\\nsub ject\\n\\nof VMLS (Figure 16.1). We use the vandermonde function from page 58.\\n\\njulia>\\n\\nM\\n\\n= 70;\\n\\nN\\n\\n=\\n\\n2*M;\\n\\njulia>\\n\\nxleft =\\n\\nrand(M)\\n\\n.-\\n\\n1;\\n\\nxright\\n\\n=\\n\\nrand(M);\\n\\njulia>\\n\\njulia>\\n\\njulia>\\n\\njulia>\\n\\nx\\n\\ny\\n\\nn\\n\\nA\\n\\n= [xleft;\\n\\nxright]\\n\\n= x.^3\\n\\n-\\n\\nx\\n\\n+\\n\\n0.4\\n\\n./\\n\\n(1\\n\\n.+\\n\\n25*x.^2)\\n\\n+\\n\\n0.05*randn(N);\\n\\n= 4;\\n\\n= [ vandermonde(xleft,n)\\n\\nzeros(M,n);\\n\\nzeros(M,n)\\n\\nvandermonde(xright,n)]\\n\\njulia>\\n\\nb = y;\\n\\njulia>\\n\\nC = [1\\n\\nzeros(1,n-1) -1\\n\\nzeros(1,n-1);\\n\\n0\\n\\n1\\n\\nzeros(1,n-2)\\n\\n0\\n\\n-1\\n\\nzeros(1,n-2)];\\n\\njulia>\\n\\nd = zeros(2);\\n\\njulia>\\n\\ntheta\\n\\n=\\n\\ncls_solve(A,\\n\\nb,\\n\\nC,\\n\\nd);\\n\\njulia>\\n\\nusing\\n\\nPlots\\n\\njulia>\\n\\n# Evaluate\\n\\nand\\n\\nplot\\n\\nfor\\n\\n200\\n\\nequidistant\\n\\npoints\\n\\non\\n\\neach\\n\\nside.\\n\\njulia>\\n\\nNpl =\\n\\n200;\\n\\njulia>\\n\\nxpl_left\\n\\n=\\n\\nlinspace(-1,\\n\\n0,\\n\\nNpl);\\n\\n\\x0c',\n",
       " \"130\\n\\n16\\n\\nConstrained least squares\\n\\nFigure 16.1 Least squares ﬁt of two cubic polynomials to 140 points, with\\ncontinuity constraints p(0) = q(0) and p(cid:48) (0) = q (cid:48) (0).\\n\\njulia> ypl_left\\n\\n=\\n\\nvandermonde(xpl_left,\\n\\n4)*theta[1:n];\\n\\njulia> xpl_right\\n\\njulia> ypl_right\\n\\n=\\n\\n=\\n\\nlinspace(0,\\n\\n1,\\n\\nNpl);\\n\\nvandermonde(xpl_right,\\n\\n4)*theta[n+1:end];\\n\\njulia> scatter(x,y,\\n\\nlegend=false)\\n\\njulia> plot!(xpl_left,\\n\\nypl_left)\\n\\njulia> plot!(xpl_right,\\n\\nypl_right)\\n\\nAdvertising budget. We continue the advertising example of page 96 and add a\\ntotal budget constraint 1T s = 1284.\\n\\njulia> cls_solve(R,\\n3×1\\n\\nArray{Float64,2}:\\n\\n1e3*ones(m),\\n\\nones(1,n),\\n\\n1284)\\n\\n315.16818459234986\\n\\n109.86643348012254\\n\\n858.9653819275276\\n\\nMinimum norm force sequence. We compute the smallest sequence of ten forces,\\n\\neach applied for one second to a unit frictionless mass originally at rest, that moves\\n\\nthe mass position one with zero velocity (VMLS page 343).\\n\\njulia> A\\n\\njulia> C\\n\\n=\\n\\n=\\n\\neye(10);\\n\\nb\\n\\n=\\n\\n[ones(1,10);\\n\\nzeros(10);\\n(9.5:-1:0.5)']\\n\\n-1.0-0.50.00.51.0-0.4-0.20.00.20.4\\x0c\",\n",
       " \"16.2\\n\\nSolution\\n\\n131\\n\\n2×10\\n\\n1.0\\n\\n9.5\\n\\nArray{Float64,2}:\\n\\n1.0\\n\\n8.5\\n\\n1.0\\n\\n7.5\\n\\n1.0\\n\\n6.5\\n\\n1.0\\n\\n5.5\\n\\n1.0\\n\\n4.5\\n\\n1.0\\n\\n3.5\\n\\n1.0\\n\\n2.5\\n\\n1.0\\n\\n1.5\\n\\n1.0\\n\\n0.5\\n\\njulia>\\n\\nd\\n\\n= [0,1];\\n\\nfln = cls_solve(A,b,C,d)\\n\\njulia>\\n10×1\\n\\nArray{Float64,2}:\\n\\n0.0545455\\n\\n0.0424242\\n\\n0.030303\\n\\n0.0181818\\n\\n0.00606061\\n\\n-0.00606061\\n\\n-0.0181818\\n\\n-0.030303\\n\\n-0.0424242\\n\\n-0.0545455\\n\\n16.2\\n\\nSolution\\n\\nLet’s\\n\\nimplement\\n\\nthe\\n\\nfunction cls_solve_kkt, which ﬁnds\\n\\nthe\\n\\nconstrained least\\n\\nsquares solution by forming the KKT system and solving it. We allow the b and\\n\\nd to be matrices, so one function call can solve mutiple problems with the same A\\n\\nand C .\\n\\njulia>\\n\\nfunction\\n\\ncls_solve_kkt(A,b,C,d)\\n\\nm, n\\n\\n=\\n\\nsize(A)\\n\\np, n\\n=\\nsize(C)\\n= A'*A\\n\\nG\\n\\n#\\n\\nKKT =\\n\\n[2*G\\n\\nGram matrix\\nC';\\n\\nC zeros(p,p)]\\n\\n#\\n\\nKKT\\n\\nmatrix\\n\\n\\x0c\",\n",
       " \"132\\n\\n16\\n\\nConstrained least squares\\n\\n16.3\\n\\nSolving contrained least squares problems\\n\\nSolving constrained least squares via QR.\\n\\nLet’s implement VMLS algorithm 16.1\\n\\nand then check it against our method above, which forms and solves\\n\\nthe KKT\\n\\nsystem.\\n\\njulia> function\\n\\ncls_solve(A,b,C,d)\\n\\nm, n\\n\\np, n\\n\\nQ, R\\n\\n=\\n\\n=\\n\\n=\\n\\nsize(A)\\n\\nsize(C)\\n\\nqr([A;\\n\\nC])\\n\\nQ\\n\\n= Matrix(Q)\\n\\nQ1 =\\n\\nQ[1:m,:]\\n\\nQ2 =\\n\\nQtil,\\n\\nQ[m+1:m+p,:]\\nqr(Q2')\\n\\nRtil =\\n\\nQtil\\n\\n=\\n\\nMatrix(Qtil)\\n\\n\\x0c\",\n",
       " '16.3\\n\\nSolving contrained least squares problems\\n\\n133\\n\\njulia>\\n\\nA\\n\\n= randn(m,n);\\n\\nb\\n\\n=\\n\\nrandn(m);\\n\\nC\\n\\n=\\n\\nrandn(p,n);\\n\\nd\\n\\n=\\n\\nrandn(p);\\n\\njulia>\\n\\nx1 =\\n\\ncls_solve(A,b,C,d);\\n\\njulia>\\n\\nx2 =\\n\\ncls_solve_sparse(A,b,C,d);\\n\\njulia>\\n\\nnorm(x1-x2)\\n\\n1.3344943251376455e-14\\n\\nSolving least norm problem.\\n\\nIn Julia, the backslash operator is used to ﬁnd the\\n\\nleast norm solution ˆx of an under-determined set of equations C x = d. Thus the\\n\\nbackslash operator is overloaded to solve linear equations with a square coeﬃcient\\n\\nmatrix, ﬁnd a least\\n\\nsquares approximate solution when the coeﬃcient matrix is\\n\\ntall, and ﬁnd the least norm solution when the coeﬃcient matrix is wide.\\n\\nLet’s\\n\\nsolve a least norm problem using several methods,\\n\\nto check that\\n\\nthey\\n\\nagree.\\n\\njulia>\\n\\njulia>\\n\\np\\n\\nC\\n\\n= 50;\\n\\nn\\n\\n=\\n\\n500;\\n\\n= randn(p,n);\\n\\nd\\n\\n=\\n\\nrandn(p);\\n\\njulia>\\n\\nx1 =\\n\\nC\\\\d;\\n\\n#\\n\\nSolve\\n\\nusing\\n\\nbackslash\\n\\njulia>\\n\\n#\\n\\nSolve\\n\\nusing\\n\\ncls_solve,\\n\\nwhich\\n\\nuses\\n\\nKKT\\n\\nsystem\\n\\njulia>\\n\\nx2 =\\n\\ncls_solve(eye(n),\\n\\nzeros(n),\\n\\nC,\\n\\nd);\\n\\njulia>\\n\\nx3 =\\n\\npinv(C)*d;\\n\\n#\\n\\nUsing\\n\\npseudo-inverse\\n\\njulia>\\n\\nnorm(x1-x2)\\n\\n5.584943800596077e-15\\n\\njulia>\\n\\nnorm(x2-x3)\\n\\n5.719694159427276e-15\\n\\n\\x0c',\n",
       " '134\\n\\n16\\n\\nConstrained least squares\\n\\n\\x0c',\n",
       " 'Chapter 17\\n\\nConstrained least squares\\n\\napplications\\n\\n17.1\\n\\nPortfolio optimization\\n\\nCompounded portfolio value.\\n\\nThe cumulative value of a portfolio from a return\\ntime series vector r , starting from the traditional value of $10000,\\n\\nis given by the\\n\\nvalue times series vector v , where\\n\\nvt = 10000(1 + r1 ) · · · (1 + rt−1 ),\\n\\nt = 1, . . . , T .\\n\\nIn other words, we form the cumulative product of the vector with entries 1 + rt .\\nJulia has a built-in function that does this, cumprod.\\n\\njulia>\\n\\n#\\n\\nPortfolio\\n\\nvalue\\n\\nwith\\n\\nre-investment,\\n\\nreturn\\n\\ntime\\n\\nseries\\n\\nr\\n\\njulia>\\n\\ncum_value(r)\\n\\n=\\n\\n10000\\n\\n*\\n\\ncumprod(1\\n\\n.+\\n\\nr)\\n\\njulia>\\n\\nT\\n\\n= 250;\\n\\n#\\n\\nOne\\n\\nyear’s\\n\\nworth\\n\\nof\\n\\ntrading\\n\\ndays\\n\\njulia>\\n\\n#\\n\\nGenerate\\n\\nrandom\\n\\nreturns\\n\\nsequence\\n\\nwith\\n\\njulia>\\n\\n#\\n\\n10% annualized\\n\\nreturn,\\n\\n5%\\n\\nannualized\\n\\nrisk\\n\\njulia>\\n\\nmu =\\n\\n0.10/250;\\n\\nsigma\\n\\n=\\n\\n0.05/sqrt(250);\\n\\njulia>\\n\\nr\\n\\n= mu .+\\n\\nsigma*randn(T);\\n\\njulia>\\n\\nv\\n\\n= cum_value(r);\\n\\njulia>\\n\\n#\\n\\ncompare\\n\\nfinal\\n\\nvalue\\n\\n(compounded)\\n\\nand\\n\\naverage\\n\\nreturn\\n\\njulia>\\n\\nv[T]\\n\\n, v[1]*(1+sum(r))\\n\\n(10313.854295827463,\\n\\n10348.11585318395)\\n\\njulia>\\n\\n#\\n\\nplot cumulative\\n\\nvalue\\n\\nover\\n\\nthe\\n\\nyear\\n\\njulia>\\n\\nusing Plots\\n\\njulia>\\n\\nplot(1:T,\\n\\nv,\\n\\nlegend=false)\\n\\njulia>\\n\\nplot!( xlabel\\n\\n=\\n\\n\"t\",\\n\\nylabel\\n\\n=\\n\\n\"v_t\")\\n\\nThe resulting ﬁgure for a particular choice of r is shown in Figure 17.1.\\n\\n\\x0c',\n",
       " \"136\\n\\n17\\n\\nConstrained least squares applications\\n\\nFigure 17.1 Total portfolio value over time.\\n\\nPortfolio optimization. We deﬁne a function port_opt that evaluates the solu-\\n\\ntion (17.3) of the constrained least squares problem (17.2) in VMLS, and apply to\\n\\nthe return data in VMLS Section 17.1.3.\\n\\njulia> function\\n\\nport_opt(R,rho)\\n\\nT, n\\n\\nmu =\\n\\nKKT =\\n\\n=\\n\\nsize(R)\\ndims=1)'/T\\nsum(R,\\n[ 2*R'*R\\n\\nones(n)\\n\\nmu;\\n\\nones(n)' 0\\n\\n0; mu' 0\\n\\n0]\\n\\n050100150200250100001010010200103001040010500tv_t\\x0c\",\n",
       " '17.2\\n\\nLinear quadratic control\\n\\n137\\n\\nFigure 17.2 Total value over\\n\\ntime for\\n\\nfour portfolios:\\n\\nthe Pareto optimal\\n\\nportfolios with 10%, 20%, and 40% return, and the uniform portfolio. The\\ntotal value is computed using the 2000 × 20 daily return matrix R.\\n\\n17.2\\n\\nLinear quadratic control\\n\\nWe implement\\n\\nlinear quadratic control, as described in VMLS §17.2,\\n\\nfor a time-\\n\\ninvariant system with matrices A, B , and C .\\n\\nKronecker product.\\n\\nTo create the big matrices\\n\\n˜A and ˜C , we need to deﬁne block\\n\\ndiagonal matrices with the\\n\\nsame matrix repeated a number of\\n\\ntimes along the\\n\\ndiagonal.\\n\\nThere are many ways\\n\\nto do this\\n\\nin Julia.\\n\\nOne of\\n\\nthe\\n\\nsimplest ways\\n\\nuses the kron function,\\nfor the Kronecker product of two matrices. The Kronecker\\nproduct of an m × n matrix G and a p × q matrix H is deﬁned as\\nthe mp × nq\\n\\nblock matrix\\n\\n\\uf8ee\\uf8ef\\uf8ef\\uf8ef\\uf8f0\\n\\nG11H\\nG21H\\n...\\n\\nG12H\\nG22H\\n...\\n\\n· · ·\\n· · ·\\n\\n· · ·\\n\\n\\uf8f9\\uf8fa\\uf8fa\\uf8fa\\uf8fb .\\n\\nG1nH\\nG2nH\\n...\\n\\nGm1H\\nGm2H\\nGmnH\\nIt is computed in Julia as kron(G,H). If G is an n × n identity matrix, we obtain\\nthe block diagonal matrix with H repeated n times on the diagonal.\\n\\njulia>\\n\\nH\\n\\n= randn(2,2)\\n\\n2x2\\n\\nArray{Float64,2}:\\n\\n1.73065\\n\\n-1.33313\\n\\n-1.52245\\n\\n0.0200201\\n\\njulia>\\nkron(eye(3),H)\\n6×6 Array{Float64,2}:\\n\\n1.73065\\n\\n-1.33313\\n\\n0.0\\n\\n-0.0\\n\\n0.0\\n\\n-0.0\\n\\n05001000150020002.50×1045.00×1047.50×1041.00×1051.25×10510%20%40%1/n\\x0c',\n",
       " '138\\n\\n17\\n\\nConstrained least squares applications\\n\\n-1.52245\\n\\n0.0200201\\n\\n-0.0\\n\\n0.0\\n\\n-0.0\\n\\n0.0\\n\\n0.0\\n\\n-0.0\\n\\n1.73065\\n\\n-1.33313\\n\\n0.0\\n\\n-0.0\\n\\n-0.0\\n\\n0.0\\n\\n-1.52245\\n\\n0.0200201\\n\\n-0.0\\n\\n0.0\\n\\n0.0\\n\\n-0.0\\n\\n0.0\\n\\n-0.0\\n\\n1.73065\\n\\n-1.33313\\n\\n-0.0\\n\\n0.0\\n\\n-0.0\\n\\n0.0\\n\\n-1.52245\\n\\n0.0200201\\n\\nAn alternative method uses the Julia cat function for constructing block matrices:\\n\\njulia> cat([H\\n6×6\\n\\nfor\\n\\nk=1:3]...,\\n\\ndims=(1,2))\\n\\nArray{Float64,2}:\\n\\n1.73065\\n\\n-1.33313\\n\\n0.0\\n\\n-0.0\\n\\n0.0\\n\\n-0.0\\n\\n-1.52245\\n\\n0.0200201\\n\\n-0.0\\n\\n0.0\\n\\n-0.0\\n\\n0.0\\n\\n0.0\\n\\n-0.0\\n\\n1.73065\\n\\n-1.33313\\n\\n0.0\\n\\n-0.0\\n\\n-0.0\\n\\n0.0\\n\\n-1.52245\\n\\n0.0200201\\n\\n-0.0\\n\\n0.0\\n\\n0.0\\n\\n-0.0\\n\\n0.0\\n\\n-0.0\\n\\n1.73065\\n\\n-1.33313\\n\\n-0.0\\n\\n0.0\\n\\n-0.0\\n\\n0.0\\n\\n-1.52245\\n\\n0.0200201\\n\\nLinear quadratic control example. We start by writing a function lqr that con-\\n\\nstructs and solves the constrained least squares problem for linear quadratic control.\\n\\nThe function returns three arrays\\n\\nx\\n\\n=\\n\\n[ x[1],\\n\\nx[2],\\n\\n...,\\n\\nx[T]\\n\\n],\\n\\nu\\n\\n=\\n\\n[ u[1],\\n\\nu[2],\\n\\n...,\\n\\nu[T-1]\\n\\n],\\n\\ny\\n\\n=\\n\\n[ y[1],\\n\\ny[2],\\n\\n...,\\n\\ny[T]\\n\\n].\\n\\nThe ﬁrst two contain the optimal solution of the problem. The third array contains\\n\\nyt = C xt .\\n\\nWe allow the input arguments x_init and x_des to be matrices, so we can solve\\n\\nthe same problem for diﬀerent pairs of initial and end states, with one function call.\\n\\nIf the number of columns in x_init and x_des is q , then the entries of the three\\n\\noutput\\n\\nsequences x, u, y are matrices with q columns. The ith columns are the\\n\\nsolution for the initial and end states speciﬁed in the ith columns of x_init and\\n\\nx_des.\\n\\nfunction\\n\\nlqr(A,B,C,x_init,x_des,T,rho)\\n\\nn\\n\\n= size(A,1)\\n\\nm\\n\\n= size(B,2)\\n\\np\\n\\n= size(C,1)\\n\\nq\\n\\n= size(x_init,2)\\n\\nAtil\\n\\n= [\\n\\nkron(eye(T),\\n\\nC)\\n\\nzeros(p*T,m*(T-1))\\n\\n;\\n\\nzeros(m*(T-1),\\n\\nn*T)\\n\\nsqrt(rho)*eye(m*(T-1))\\n\\n]\\n\\nbtil\\n\\n= zeros(p*T\\n\\n+\\n\\nm*(T-1),\\n\\nq)\\n\\n#\\n\\nWe’ll construct\\n\\nCtilde\\n\\nbit\\n\\nby\\n\\nbit\\n\\nCtil11 =\\n\\n[\\n\\nkron(eye(T-1),\\n\\nA)\\n\\nzeros(n*(T-1),n)\\n\\n]\\n\\n-\\n\\n\\x0c',\n",
       " \"17.2\\n\\nLinear quadratic control\\n\\n139\\n\\n[\\n\\nzeros(n*(T-1),\\n\\nn)\\n\\neye(n*(T-1))\\n\\n]\\n\\nCtil12 =\\n\\nkron(eye(T-1), B)\\n\\nCtil21 =\\n\\n[eye(n)\\n\\nzeros(n,n*(T-1));\\n\\nzeros(n,n*(T-1))\\n\\neye(n)]\\n\\nCtil22 =\\n\\nzeros(2*n,m*(T-1))\\n\\nCtil\\n\\n= [Ctil11\\n\\nCtil12;\\n\\nCtil21\\n\\nCtil22]\\n\\ndtil\\n\\n= [zeros(n*(T-1),\\n\\nq);\\n\\nx_init;\\n\\nx_des]\\n\\nz\\n\\n= cls_solve(Atil,btil,Ctil,dtil)\\n\\nx\\n\\n= [z[(i-1)*n+1:i*n,:] for\\n\\ni=1:T]\\n\\nu\\n\\n= [z[n*T+(i-1)*m+1\\n\\n:\\n\\nn*T+i*m,\\n\\n:]\\n\\nfor\\n\\ni=1:T-1]\\n\\ny\\n\\n= [C*xt\\n\\nfor\\n\\nxt\\n\\nin\\n\\nx]\\n\\nreturn x,\\n\\nu, y\\n\\nend;\\n\\nWe apply the function to the example in §17.2.1.\\n\\njulia>\\n\\nA\\n\\n= [ 0.855\\n\\n1.161\\n\\n0.667;\\n\\n0.015\\n\\n1.073\\n\\n0.053;\\n\\n-0.084\\n\\n0.059\\n\\n1.022\\n\\n];\\n\\njulia>\\n\\nB = [-0.076;\\n\\n-0.139;\\n\\n0.342\\n\\n];\\n\\njulia>\\n\\nC = [\\n\\n0.218\\n\\n-3.597\\n\\n-1.683\\n\\n];\\n\\njulia>\\n\\nn = 3;\\n\\np\\n\\n=\\n\\n1;\\n\\nm\\n\\n=\\n\\n1;\\n\\njulia>\\n\\nx_init\\n\\n=\\n\\n[0.496;\\n\\n-0.745;\\n\\n1.394];\\n\\njulia>\\n\\nx_des\\n\\n=\\n\\nzeros(n,1);\\n\\nWe ﬁrst plot the open-loop response of VMLS ﬁgure 17.4 in ﬁgure 17.3.\\n\\njulia>\\n\\nT\\n\\n= 100;\\n\\njulia>\\n\\nyol = zeros(T,1);\\n\\njulia>\\n\\nXol = [\\n\\nx_init\\n\\nzeros(n,\\n\\nT-1)\\n\\n];\\n\\njulia>\\n\\nfor k=1:T-1\\n\\nXol[:,k+1]\\n\\n= A*Xol[:,k];\\n\\nend;\\n\\njulia> yol\\n\\n= C*Xol;\\n\\njulia> using Plots\\nyol',\\n\\njulia> plot(1:T,\\n\\nlegend\\n\\n=\\n\\nfalse)\\n\\nWe then solve the linear quadratic control problem with T = 100 and ρ = 0.2. The\\n\\nresult is shown in the second row of VMLS ﬁgure 17.6 and in ﬁgure 17.4.\\n\\njulia>\\n\\nrho = 0.2;\\n\\njulia>\\n\\nT\\n\\n= 100;\\n\\njulia>\\n\\nx, u, y\\n\\n=\\n\\nlqr(A,B,C,x_init,x_des,T,rho)\\n\\njulia>\\n\\nJ_input\\n\\n=\\n\\nnorm(u)^2\\n\\n\\x0c\",\n",
       " '140\\n\\n17\\n\\nConstrained least squares applications\\n\\nFigure 17.3 Open-loop response CAt−1xinit .\\n\\n0.7738942551160318\\n\\njulia> J_output\\n\\n=\\n\\nnorm(y)^2\\n\\n3.7829986463323224\\n\\njulia> plot(1:T-1,\\n\\nvcat(u...),\\n\\nlegend\\n\\n=\\n\\nfalse,\\n\\nxlabel=\"t\",\\n\\nylabel=\\n\\n\"u_t\")\\n\\njulia> plot(1:T,\\n\\nvcat(y...),\\n\\nlegend=false,\\n\\nxlabel =\\n\\n\"t\",\\n\\nylabel =\\n\\n\"y_t\")\\n\\nLinear\\n\\nstate feedback control.\\n\\nTo ﬁnish the\\n\\nexample we\\n\\nimplement\\n\\nthe\\n\\nstate\\n\\nfeedback method in VMLS section 17.2.3. The plots in ﬁgure 17.5 reproduce VMLS\\n\\nﬁgure 17.7.\\n\\njulia> #\\n\\nSolve\\n\\nLQ\\n\\nproblem\\n\\nwith\\n\\nx_init\\n\\n=\\n\\nI,\\n\\nx_des =\\n\\n0\\n\\njulia> rho\\n\\n= 1.0;\\n\\njulia> xsf, usf,\\n\\nysf\\n\\n=\\n\\nlqr(A,B,C,eye(n),zeros(n,n),T,rho);\\n\\njulia> K\\n\\n=\\n\\nusf[1];\\n\\njulia> #\\n\\nSimulate\\n\\nover\\n\\nhorizon\\n\\n150\\n\\njulia> TT\\n\\n= 150;\\n\\njulia> Xsf\\n\\n= [x_init\\n\\nzeros(n,TT-1)];\\n\\njulia> for\\n\\nk=1:TT-1\\n\\nXsf[:,k+1]\\n\\n=\\n\\n(A+B*K)*Xsf[:,k];\\n\\nend;\\n\\njulia> usf\\n\\n= K*Xsf[:,\\n\\n1:TT-1];\\n\\njulia> ysf\\n\\n= C*Xsf;\\n\\njulia> #\\n\\nAlso\\n\\ncompute\\n\\noptimal\\n\\nLQ\\n\\nsolution\\n\\nfor\\n\\nrho =\\n\\n1.0\\n\\n02550751000.00.10.20.30.4\\x0c',\n",
       " '17.2\\n\\nLinear quadratic control\\n\\n141\\n\\nFigure 17.4 Optimal\\n\\ninput and output for ρ = 0.2.\\n\\n0255075100-0.2-0.10.00.10.20.3tu_t02550751000.00.10.20.30.4ty_t\\x0c',\n",
       " '142\\n\\n17\\n\\nConstrained least squares applications\\n\\nFigure 17.5 The blue curves are the solutions of the linear quadratic control\\n\\nproblem for ρ = 1. The red curves are the inputs and outputs that result\\n\\nfrom the constant state feedback ut = K xt .\\n\\n.\\n\\njulia> x,\\n\\nu, y\\n\\n=\\n\\nlqr(A,B,C,x_init,x_des,T,rho)\\n\\njulia> #\\n\\nPlot\\n\\nthe\\n\\ntwo\\n\\ninputs\\n\\njulia> plot([vcat(u...);\\n\\nzeros(TT-T,1)],\\n\\nlabel=\"Optimal\",\\njulia> plot!(usf\\',\\n\\nlabel\\n\\n=\\n\\n\"State\\n\\nfeedback\")\\n\\nxlabel\\n\\n=\\n\\n\"t\",\\n\\nylabel\\n\\n=\\n\\n\"u_t\")\\n\\njulia> #\\n\\nPlot\\n\\nthe\\n\\ntwo\\n\\noutputs\\n\\njulia> plot_sf_y\\n\\n=\\n\\nplot([vcat(y...);\\n\\nzeros(TT-T,1)],\\n\\nlabel=\"Optimal\",\\njulia> plot!(ysf\\',\\n\\nlabel\\n\\n=\\n\\n\"State\\n\\nfeedback\")\\n\\nxlabel\\n\\n=\\n\\n\"t\",\\n\\nylabel\\n\\n=\\n\\n\"y_t\")\\n\\n050100150-0.10-0.050.000.050.10tu_tOptimalState feedback0501001500.00.10.20.30.4ty_tOptimalState feedback\\x0c',\n",
       " '17.3\\n\\nLinear quadratic state estimation\\n\\n143\\n\\n17.3\\n\\nLinear quadratic state estimation\\n\\nThe code for the linear quadratic estimation method is very similar to the one for\\n\\nlinear quadratic control.\\n\\nfunction\\n\\nlqe(A,B,C,y,T,lambda)\\n\\nn\\n\\n= size(A,1)\\n\\nm\\n\\n= size(B,2)\\n\\np\\n\\n= size(C,1)\\n\\nAtil\\n\\n= [\\n\\nkron(eye(T),\\n\\nC)\\n\\nzeros(T*p,\\n\\nm*(T-1));\\n\\nzeros(m*(T-1),\\n\\nn*T)\\n\\nsqrt(lambda)*eye(m*(T-1))\\n\\n]\\n\\n#\\n\\nWe\\n\\nassume y\\n\\nis\\n\\na\\n\\np\\n\\nx\\n\\nT\\n\\narray,\\n\\nso\\n\\nwe\\n\\nvectorize\\n\\nit\\n\\nbtil\\n\\n= [\\n\\nvcat(y...)\\n\\n;\\n\\nzeros((T-1)*m)\\n\\n]\\n\\nCtil\\n\\n= [\\n\\n([ kron(eye(T-1),\\n\\nA)\\n\\nzeros(n*(T-1),\\n\\nn)\\n\\n]\\n\\n+\\n\\n[\\n\\nzeros(n*(T-1),\\n\\nn)\\n\\n-eye(n*(T-1))\\n\\n])\\n\\nkron(eye(T-1),\\n\\nB)\\n\\n]\\n\\ndtil\\n\\n= zeros(n*(T-1))\\n\\nz\\n\\n= cls_solve(Atil,\\n\\nbtil,\\n\\nCtil,\\n\\ndtil)\\n\\nx\\n\\n= [\\n\\nz[(i-1)*n+1:i*n]\\n\\nfor\\n\\ni=1:T\\n\\n]\\n\\nu\\n\\n= [\\n\\nz[n*T+(i-1)*m+1\\n\\n: n*T+i*m]\\n\\nfor\\n\\ni=1:T-1\\n\\n]\\n\\ny\\n\\n= [\\n\\nC*xt\\n\\nfor\\n\\nxt\\n\\nin\\n\\nx\\n\\n]\\n\\nreturn x,\\n\\nu, y\\n\\nend\\n\\nin §17.3.1 of VMLS. The output measurement data\\nare read from an input ﬁle estimation_data.jl, which creates a 2 × 100 matrix\\nWe use the system matrices\\nymeas. We compute the solution for λ = 103 , shown in the lower-left plot of VMLS\\n\\nﬁgure 17.8.\\n\\njulia>\\n\\nymeas =\\n\\nlq_estimation_data();\\n\\njulia>\\n\\nA\\n\\n= [ eye(2)\\n\\neye(2);\\n\\nzeros(2,2)\\n\\neye(2)\\n\\n];\\n\\njulia>\\n\\nB\\n\\n= [ zeros(2,2);\\n\\neye(2)\\n\\n];\\n\\njulia>\\n\\nC\\n\\n= [ eye(2)\\n\\nzeros(2,2)\\n\\n];\\n\\njulia>\\n\\nn\\n\\n= size(A,1);\\n\\njulia>\\n\\nm\\n\\n= size(B,2);\\n\\njulia>\\n\\np\\n\\n= size(C,1);\\n\\njulia>\\n\\nT\\n\\n= 100;\\n\\njulia>\\n\\nlambda =\\n\\n1e3;\\n\\njulia>\\n\\nxest, uest,\\n\\nyest\\n\\n=\\n\\nlqe(A,B,C,y,T,lambda)\\n\\njulia>\\n\\nusing Plots\\n\\njulia>\\n\\nscatter(y[1,:],\\n\\ny[2,:],\\n\\nlegend\\n\\n=\\n\\nfalse,\\n\\naxis\\n\\n=\\n\\nfalse)\\n\\njulia>\\n\\nplot!( [yt[1]\\n\\nfor\\n\\nyt\\n\\nin\\n\\nyest],\\n\\n[yt[2]\\n\\nfor\\n\\nyt\\n\\nin\\n\\nyest])\\n\\nThe result can be seen in ﬁgure 17.6\\n\\n\\x0c',\n",
       " '144\\n\\n17\\n\\nConstrained least squares applications\\n\\nFigure 17.6 The circles show 100 noisy measurements in 2-D. The solid line\\n\\nis the estimated tra jectory C ˆxt\\n\\nfor λ = 1000.\\n\\n\\x0c',\n",
       " 'Chapter 18\\n\\nNonlinear least squares\\n\\n18.1\\n\\nNonlinear equations and least squares\\n\\n18.2\\n\\nGauss–Newton algorithm\\n\\nBasic Gauss–Newton algorithm.\\n\\nLet’s ﬁrst implement the basic Gauss–Newton\\n\\nmethod (algorithm 18.1 in VMLS) in Julia.\\n\\nIn Julia, you can pass a function as\\n\\nan argument to another function, so we can pass f (the function) and also Df (the\\n\\nderivative or Jacobian matrix) to our Gauss–Newton algorithm.\\n\\nfunction gauss_newton(f,\\n\\nDf,\\n\\nx1;\\n\\nkmax\\n\\n=\\n\\n10)\\n\\nx\\n\\n= x1\\n\\nfor k\\n\\n= 1:kmax\\n\\n1\\n\\n2\\n\\n3\\n\\n4\\n\\n\\x0c',\n",
       " '146\\n\\n18\\n\\nNonlinear least squares\\n\\nfunction\\n\\nnewton(f,\\n\\nDf,\\n\\nx1;\\n\\nkmax\\n\\n=\\n\\n20,\\n\\ntol\\n\\n=\\n\\n1e-6)\\n\\nx\\n\\n= x1\\n\\nfnorms =\\n\\nzeros(1,0)\\n\\nfor k\\n\\n= 1:kmax\\n\\nfk =\\n\\nf(x)\\n\\nfnorms =\\n\\n[fnorms;\\n\\nnorm(fk)]\\n\\nif norm(fk)\\n\\n< tol\\n\\nbreak\\n\\nend;\\n\\n1\\n\\n2\\n\\n3\\n\\n4\\n\\n5\\n\\n6\\n\\n7\\n\\n8\\n\\n9\\n\\n10\\n\\n\\x0c',\n",
       " '18.3\\n\\nLevenberg–Marquardt algorithm\\n\\n147\\n\\nFigure 18.1 The ﬁrst iterations in the Newton algorithm for solving f (x) = 0\\n\\nfor starting point x(1) = 0.95.\\n\\nThe method converges very quickly, as can also be seen in ﬁgure 18.1. However it\\ndoes not converge for a slightly larger starting point x(1) = 1.15.\\n\\njulia>\\n\\nx, fnorms\\n\\n=\\n\\nnewton(f,Df,1.15);\\n\\njulia>\\n\\nf(x)\\n\\nNaN\\n\\njulia>\\n\\nfnorms[1:5]\\n\\n5-element Array{Float64,2}:\\n\\n0.8177540779702877\\n\\n0.8664056534177534\\n\\n0.9735568532451108\\n\\n0.9999999999999906\\n\\nNaN\\n\\n18.3\\n\\nLevenberg–Marquardt algorithm\\n\\nThe Gauss–Newton algorithm can fail\\nif\\nthe derivative matrix does not have in-\\nIt also does not guarantee that (cid:107)f (x(k) )(cid:107) decreases\\n\\ndependent columns.\\n\\nin each\\n\\niteration. Both of these shortcomings are addressed in the Levenberg–Marquardt\\n\\nalgorithm. Below is a Julia implementation of algorithm 18.3 in VMLS. This func-\\n\\ntion is included in the Vmls package.\\n\\n123450.00.20.40.6k|f|\\x0c',\n",
       " \"148\\n\\n18\\n\\nNonlinear least squares\\n\\nfunction\\n\\nlevenberg_marquardt(f,\\n\\nDf,\\n\\nx1,\\n\\nlambda1;\\n\\nkmax=100,\\n\\ntol=1e-6)\\n\\nn\\n\\nx\\n\\n= length(x1)\\n\\n= x1\\n\\nlambda =\\n\\nlambda1\\n\\nobjectives\\n\\n=\\n\\nzeros(0,1)\\n\\nresiduals\\n\\n=\\n\\nzeros(0,1)\\n\\nfor k\\n\\n= 1:kmax\\n\\n[objectives;\\nnorm(fk)^2]\\nnorm(2*Dfk'*fk)]\\n\\nresiduals =\\n[residuals;\\nif norm(2*Dfk'*fk) <\\n\\ntol\\n\\nfk =\\n\\nf(x)\\n\\nDfk =\\n\\nDf(x)\\n\\nobjectives =\\n\\nbreak\\n\\nend;\\n\\n1\\n\\n2\\n\\n3\\n\\n4\\n\\n5\\n\\n6\\n\\n7\\n\\n8\\n\\n9\\n\\n10\\n\\n11\\n\\n12\\n\\n13\\n\\n14\\n\\n15\\n\\n\\x0c\",\n",
       " '18.3\\n\\nLevenberg–Marquardt algorithm\\n\\n149\\n\\n18.2\\n\\nFigure\\n\\n|f (x(k) )|\\nnumber\\nk\\nfor\\n(exp(x) −\\nthe\\nLevenberg–Marquardt\\nalgorithm applied\\nto\\nf (x)\\n=\\nexp(−x))/(exp(x)+exp(−x)). The starting point is x(1) = 1.15 and λ(1) = 1.\\n\\niteration\\n\\nValues\\n\\nversus\\n\\nthe\\n\\nof\\n\\nlegend =\\n\\nfalse,\\n\\nxlabel\\n\\n=\\n\\n\"k\",\\n\\nylabel\\n\\n=\\n\\n\"|f|\");\\n\\nThe result is shown in ﬁgure 18.2.\\nNote that we deﬁned x(1) as the array [1.15], and use dot-operations in the\\n\\ndeﬁnitions of f and Df to ensure that these functions work with vector arguments.\\n\\nThis is important because Julia distinguishes between scalars and 1-vectors.\\n\\nIf we\\n\\ncall\\n\\nthe levenberg_marquardt function with a scalar argument x1,\\n\\nline 15 will\\n\\nraise an error, because Julia does not accept subtractions of scalars and 1-vectors.\\n\\nEquilibrium prices. We solve a nonlinear equation f (p) = 0 with two variables,\\n\\nwhere\\n\\nf (p) = exp(E s log p + snom ) − exp(E d log p + dnom ).\\n\\n(18.2)\\n\\nHere exp and log are interpreted as element-wise vector operations. The problem\\n(cid:20) −0.5\\nparameters are snom = (2.2, 0.3), dnom = (3.1, 2.2),\\n0\\n\\n0.5\\n−0.15\\n\\n0.2\\n−0.5\\n\\n−0.3\\n\\nE d =\\n\\nE s =\\n\\n(cid:20)\\n\\n(cid:21)\\n\\n0.8\\n\\n,\\n\\n(cid:21)\\n\\n.\\n\\njulia>\\n\\nsnom\\n\\n= [2.2,\\n\\n0.3];\\n\\njulia>\\n\\ndnom\\n\\n= [3.1,\\n\\n2.2];\\n\\njulia>\\n\\nEs =\\n\\n[0.5\\n\\n-.3;\\n\\n-0.15\\n\\n0.8];\\n\\njulia>\\n\\nEd =\\n\\n[-0.5\\n\\n0.2;\\n\\n-0.00\\n\\n-0.5];\\n\\njulia>\\n\\nf(p)\\n\\n= exp.(Es\\n\\n*\\n\\nlog.(p)\\n\\n+\\n\\nsnom)\\n\\n-\\n\\nexp.(Ed\\n\\n*\\n\\nlog.(p)\\n\\n+\\n\\ndnom);\\n\\njulia>\\n\\nfunction\\n\\nDf(p)\\n\\n2468100.00.20.40.60.8k|f|\\x0c',\n",
       " '150\\n\\n18\\n\\nNonlinear least squares\\n\\nFigure 18.3 Cost\\n\\nfunction (cid:107)f (p(k) (cid:107)2 versus\\nample of equation (18.2).\\n\\niteration number k for\\n\\nthe ex-\\n\\nS\\n\\nD\\n\\n= exp.(Es\\n\\n= exp.(Ed\\n\\n*\\n\\n*\\n\\nlog.(p)\\n\\nlog.(p)\\n\\n+\\n\\n+\\n\\nsnom);\\n\\ndnom);\\n\\nreturn\\n\\n[ S[1]*Es[1,1]/p[1]\\n\\nS[1]*Es[1,2]/p[2];\\n\\nS[2]*Es[2,1]/p[1]\\n\\nS[2]*Es[2,2]/p[2]\\n\\n]\\n\\n-\\n\\n[\\n\\nD[1]*Ed[1,1]/p[1]\\n\\nD[1]*Ed[1,2]/p[2];\\n\\nD[2]*Ed[2,1]/p[1]\\n\\nD[2]*Ed[2,2]/p[2]\\n\\n];\\n\\nend;\\n\\njulia>\\n\\np,\\n\\nhistory\\n\\n=\\n\\nlevenberg_marquardt(f,\\n\\nDf,\\n\\n[3,\\n\\n9],\\n\\n1);\\n\\njulia>\\n\\np\\n\\n2-element\\n\\nArray{Float64,1}:\\n\\n5.379958476145877\\n\\n4.996349602562754\\n\\njulia>\\n\\nusing\\n\\nPlots\\n\\njulia>\\n\\nplot(history[\"objectives\"],\\n\\nshape\\n\\n=\\n\\n:circle,\\n\\nlegend\\n\\n=false,\\n\\nxlabel\\n\\n=\\n\\n\"k\",\\n\\nylabel\\n\\n=\\n\\n\"Objective\")\\n\\nFigure 18.3 shows the plot of (cid:107)f (p(k) )(cid:107)2 versus iteration number k .\\n\\nLocation from range measurements.\\n\\nThe next\\n\\nexample\\n\\nis\\n\\nthe\\n\\nlocation from\\n\\nrange measurements problem on page 396 in VMLS. The positions of the m = 5\\npoints ai are given as rows in a 5 × 2 matrix A. The measurements are given in a\\n5-vector rhos. To simplify the code for\\nfunction dist(x) that computes the vector of distances ((cid:107)x − a1(cid:107), . . . , (cid:107)x − am (cid:107)).\\nthe functions f(x) and Df(x) we add a\\n\\n2.55.07.510.012.50255075100125kObjective\\x0c',\n",
       " '18.3\\n\\nLevenberg–Marquardt algorithm\\n\\n151\\n\\nThe expression for the derivative is\\n\\nDf (x) =\\n\\n\\uf8ee\\uf8ef\\uf8ef\\uf8ef\\uf8ef\\uf8ef\\uf8f0\\n\\nx1 − (a1 )1\\n\\n(cid:107)x − a1 (cid:107)\\n\\nx2 − (a1 )2\\n\\n(cid:107)x − a1(cid:107)\\n\\n...\\nx1 − (am )1\\n\\n...\\nx2 − (am ))2\\n\\n(cid:107)x − am (cid:107)\\n\\n(cid:107)x − am(cid:107)\\n\\n\\uf8f9\\uf8fa\\uf8fa\\uf8fa\\uf8fa\\uf8fa\\uf8fb .\\n\\nThis can be evaluated as\\nthe product of a diagonal matrix with diagonal entries\\n1/(cid:107)x − ai (cid:107) and the 5 × 2 matrix with i, j entry (x − ai )j .\\nWe run the Levenberg–Marquardt method for three starting points and λ(1) =\\n\\n0.1. The plot is shown in ﬁgure 18.4.\\n\\njulia>\\n\\n#\\n\\nFive locations\\n\\nai\\n\\nin\\n\\na\\n\\n5x2\\n\\nmatrix.\\n\\njulia>\\n\\nA\\n\\n= [ 1.8\\n\\n2.5;\\n\\n2.0\\n\\n1.7;\\n\\n1.5\\n\\n1.5;\\n\\n1.5\\n\\n2.0;\\n\\n2.5\\n\\n1.5\\n\\n];\\n\\njulia>\\n\\n#\\n\\nVector\\n\\nof\\n\\nmeasured\\n\\ndistances\\n\\nto\\n\\nfive\\n\\nlocations.\\n\\njulia>\\n\\nrhos\\n\\n= [\\n\\n1.87288,\\n\\n1.23950,\\n\\n0.53672,\\n\\n1.29273,\\n\\n1.49353\\n\\n];\\n\\njulia>\\n\\n#\\n\\ndist(x)\\n\\nreturns\\n\\na\\n\\n5-vector\\n\\nwith\\n\\nthe\\n\\ndistances\\n\\n||x-ai||.\\n\\njulia>\\n\\ndist(x)\\n\\n=\\n\\nsqrt.(\\n\\n(x[1]\\n\\n.-\\n\\nA[:,1]).^2\\n\\n+\\n\\n(x[2]\\n\\n.-\\n\\nA[:,2]).^2\\n\\n);\\n\\njulia>\\n\\n#\\n\\nf(x) returns\\n\\nthe\\n\\nfive\\n\\nresiduals.\\n\\njulia>\\n\\nf(x)\\n\\n= dist(x)\\n\\n-\\n\\nrhos;\\n\\njulia>\\n\\n#\\n\\nDf(x)\\n\\nis\\n\\nthe\\n\\n5x2\\n\\nderivative.\\n\\njulia>\\n\\nDf(x) =\\n\\ndiagonal(1\\n\\n./\\n\\ndist(x))\\n\\n*\\n\\n[\\n\\n(x[1] .-\\n\\nA[:,1])\\n\\n(x[2]\\n\\n.-\\n\\nA[:,2])\\n\\n];\\n\\njulia>\\n\\n# Solve\\n\\nwith\\n\\nstarting\\n\\npoint\\n\\n(1.8,3.5)\\n\\nand lambda\\n\\n=\\n\\n0.1.\\n\\njulia>\\n\\nx1, history1\\n\\n=\\n\\nlevenberg_marquardt(f,\\n\\nDf, [1.8,\\n\\n3.5],\\n\\n0.1);\\n\\njulia>\\n\\nx1\\n\\n2-element\\n\\nArray{Float64,1}:\\n\\n1.1824859803827907\\n\\n0.8242289367900364\\n\\njulia>\\n\\n# Starting\\n\\npoint\\n\\n(3.0,1.5).\\n\\njulia>\\n\\nx2, history2\\n\\n=\\n\\nlevenberg_marquardt(f,\\n\\nDf, [3.0,\\n\\n1.5],\\n\\n0.1);\\n\\njulia>\\n\\nx2\\n\\n2-element\\n\\nArray{Float64,1}:\\n\\n1.1824857942435818\\n\\n0.8242289466379732\\n\\njulia>\\n\\n# Starting\\n\\npoint\\n\\n(2.2,3.5).\\n\\njulia>\\n\\nx3, history3\\n\\n=\\n\\nlevenberg_marquardt(f,\\n\\nDf, [2.2,\\n\\n3.5],\\n\\n0.1);\\n\\njulia>\\n\\nx3\\n\\n2-element\\n\\nArray{Float64,1}:\\n\\n2.9852664103617954\\n\\n2.1215768036188956\\n\\njulia>\\n\\nusing\\n\\nPlots\\n\\n\\x0c',\n",
       " '152\\n\\n18\\n\\nNonlinear least squares\\n\\nFigure 18.4 Cost function (cid:107)f (x(k) )(cid:107)2 versus iteration number k for the three\\nstarting points in the location from range measurements example.\\n\\njulia> plot(history1[\"objectives\"][1:10],\\n\\nshape\\n\\n= :circle,\\n\\nlabel\\n\\n=\\n\\n\"Starting\\n\\npoint\\n\\n1\")\\n\\njulia> plot!(history2[\"objectives\"][1:10],\\n\\nshape =\\n\\n:circle,\\n\\nlabel\\n\\n=\\n\\n\"Starting\\n\\npoint\\n\\n2\")\\n\\njulia> plot!(history3[\"objectives\"][1:10],\\n\\nshape =\\n\\n:circle,\\n\\nlabel\\n\\n=\\n\\n\"Starting\\n\\npoint\\n\\n3\")\\n\\njulia> plot!(xlabel\\n\\n=\\n\\n\"k\",\\n\\nylabel\\n\\n=\\n\\n\"Objective\")\\n\\n18.4\\n\\nNonlinear model ﬁtting\\n\\nExample. We ﬁt a model\\n\\nˆf (x; θ) = θ1 eθ2 x cos(θ3x + θ4 )\\n\\nto N = 60 data points. We ﬁrst generate the data.\\n\\njulia> #\\n\\nUse these\\n\\nparameters\\n\\nto\\n\\ngenerate\\n\\ndata.\\n\\njulia> theta_ex\\n\\n=\\n\\n[1,\\n\\n-0.2,\\n\\n2*pi/5,\\n\\npi/3];\\n\\njulia> #\\n\\nChoose\\n\\n60\\n\\npoints\\n\\nx\\n\\nbetween\\n\\n0\\n\\nand\\n\\n20.\\n\\njulia> M\\n\\n=\\n\\n30;\\n\\njulia> xd\\n\\n= [5*rand(M);\\n\\n5\\n\\n.+\\n\\n15*rand(M)];\\n\\n2468100123kObjectiveStarting point 1Starting point 2Starting point 3\\x0c',\n",
       " '18.4\\n\\nNonlinear model ﬁtting\\n\\n153\\n\\njulia>\\n\\n#\\n\\nEvaluate\\n\\nfunction\\n\\nat\\n\\nthese\\n\\npoints.\\n\\njulia>\\n\\nyd =\\n\\ntheta_ex[1]\\n\\n*\\n\\nexp.(theta_ex[2]*xd)\\n\\n.*\\n\\ncos.(theta_ex[3]\\n\\n*\\n\\nxd\\n\\n.+\\n\\ntheta_ex[4])\\n\\njulia> #\\n\\nCreate\\n\\na\\n\\nrandom\\n\\nperturbation\\n\\nof\\n\\nyd.\\n\\njulia> N\\n\\n=\\n\\nlength(xd);\\n\\njulia> yd\\n\\n= yd\\n\\n.*\\n\\n(1\\n\\n.+\\n\\n0.2*randn(N))\\n\\n.+\\n\\n0.015\\n\\n* randn(N);\\n\\njulia> #\\n\\nPlot\\n\\ndata\\n\\npoints.\\n\\njulia> using Plots\\n\\njulia> scatter(xd,\\n\\nyd,\\n\\nlegend=false)\\n\\nThe 60 points are shown in ﬁgure 18.5. We now run our Levenberg–Marquardt\\ncode with starting point θ(1) = (1, 0, 1, 0) and λ(1) = 1. The ﬁtted model\\n\\nis shown\\n\\nin ﬁgure 18.5.\\n\\njulia>\\n\\nf(theta)\\n\\n=\\n\\ntheta[1]\\n\\n*\\n\\nexp.(theta[2]*xd)\\n\\n.*\\n\\ncos.(theta[3]\\n\\n*\\n\\nxd\\n\\n.+\\n\\ntheta[4])\\n\\n-\\n\\nyd;\\n\\njulia> Df(theta)\\n\\n=\\n\\nhcat(\\n\\nexp.(theta[2]*xd)\\n\\n.*\\n\\ncos.(theta[3]\\n\\n*\\n\\nxd\\n\\n.+\\n\\ntheta[4]),\\n\\ntheta[1]\\n\\n*\\n\\n( xd\\n\\n.*\\n\\nexp.(theta[2]*xd)\\n\\n.*\\n\\ncos.(theta[3]\\n\\n*\\n\\nxd\\n\\n.+\\n\\ntheta[4])),\\n\\n-theta[1]\\n\\n* (\\n\\nexp.(theta[2]*xd)\\n\\n.*\\n\\nxd\\n\\n.*\\n\\nsin.(theta[3]\\n\\n*\\n\\nxd\\n\\n.+\\n\\ntheta[4])),\\n\\n-theta[1]\\n\\n* (\\n\\nexp.(theta[2]*xd)\\n\\n.*\\n\\nsin.(theta[3]\\n\\n*\\n\\nxd\\n\\n.+\\n\\ntheta[4]))\\n\\n);\\n\\njulia>\\n\\ntheta1 =\\n\\n[1,\\n\\n0,\\n\\n1,\\n\\n0];\\n\\njulia>\\n\\ntheta, history\\n\\n=\\n\\nlevenberg_marquardt(f,\\n\\nDf,\\n\\ntheta1,\\n\\n1.0)\\n\\njulia>\\n\\ntheta\\n\\n4-element Array{Float64,1}:\\n\\n1.0065969737811806\\n\\n-0.23115179954434736\\n\\n1.2697087881931268\\n\\n1.0133243392186635\\n\\njulia>\\n\\n#\\n\\nPlot the\\n\\nfitted\\n\\nmodel.\\n\\njulia>\\n\\nx\\n\\n= linspace(0,\\n\\n20,\\n\\n500);\\n\\njulia>\\n\\ny=theta[1]*exp.(theta[2]*x)\\n\\n.*\\n\\ncos.(theta[3]*x\\n\\n.+\\n\\ntheta[4]);\\n\\njulia>\\n\\nplot!(x,\\n\\ny,\\n\\nlegend\\n\\n=\\n\\nfalse);\\n\\nOrthogonal distance regression.\\n\\nIn ﬁgure 18.14 of VMLS we use orthogonal dis-\\n\\ntance regression to ﬁt a cubic polynomial\\n\\nˆf (x; θ) = θ1 + θ2x + θ3x2 + θ4x3\\n\\nto N = 25 data points.\\n\\nWe ﬁrst read in the data and compute the standard least squares ﬁt.\\n\\n\\x0c',\n",
       " '154\\n\\n18\\n\\nNonlinear least squares\\n\\nFigure 18.5 Least squares ﬁt of a function ˆf (x; θ) = θ1 eθ2 x cos(θ3x + θ4 ) to\\nN = 60 points (x(i) , y (i) ).\\n\\njulia> xd,\\n\\nyd\\n\\n=\\n\\north_dist_reg_data();\\n\\n#\\n\\n2\\n\\nvectors\\n\\nof\\n\\nlength\\n\\nN\\n\\n=\\n\\n25\\n\\njulia> N\\n\\njulia> p\\n\\n=\\n\\n=\\n\\njulia>\\n\\nlength(xd);\\n\\n4;\\n\\n05101520-0.50-0.250.000.250.50\\x0c',\n",
       " '18.5\\n\\nNonlinear least squares classiﬁcation\\n\\n155\\n\\nfunction\\n\\nDf(x)\\n\\ntheta\\n\\n= x[1:p]\\n\\nu\\n\\n= x[p+1:end]\\n\\nD11 =\\n\\nvandermonde(u,p)\\n\\nD12 =\\n\\ndiagonal(theta[2] .+\\n\\n2*theta[3]*u\\n\\n.+\\n\\n3*theta[4]*u.^2)\\n\\nD21 =\\n\\nzeros(N,p)\\n\\nD22 =\\n\\neye(N)\\n\\nreturn [\\n\\nD11 D12;\\n\\nD21\\n\\nD22]\\n\\nend\\n\\nWe now call levenberg_marquardt with these two functions. A natural choice for\\n\\nthe initial point is to use the least squares solution for the variables θ and the data\\npoints x(i)\\nfor the variables u(i) . We use λ(1) = 0.01.\\n\\njulia>\\n\\nsol,\\n\\nhist\\n\\n=\\n\\nlevenberg_marquardt(f,\\n\\nDf,\\n\\n[theta_ls;\\n\\nxd],\\n\\n0.01);\\n\\njulia>\\n\\ntheta_od\\n\\n=\\n\\nsol[1:p];\\n\\nFigure 18.6 shows the two ﬁtted polynomials.\\n\\njulia>\\n\\nusing Plots\\n\\njulia>\\n\\nscatter(xd,yd,\\n\\nlabel=\"\",\\n\\nlegend\\n\\n=\\n\\n:topleft);\\n\\njulia>\\n\\nx\\n\\n= linspace(minimum(xd),\\n\\nmaximum(xd),\\n\\n500);\\n\\njulia>\\n\\ny_ls\\n\\n= vandermonde(x,\\n\\njulia>\\n\\ny_od\\n\\n= vandermonde(x,\\n\\np)\\n\\np)\\n\\n*\\n\\n*\\n\\ntheta_ls;\\n\\ntheta_od;\\n\\njulia>\\n\\nplot!(x,\\n\\ny_ls,\\n\\nlabel\\n\\njulia>\\n\\nplot!(x,\\n\\ny_od,\\n\\nlabel\\n\\n=\\n\\n=\\n\\n\"LS\");\\n\\n\"Orth.\\n\\ndist.\");\\n\\n18.5\\n\\nNonlinear least squares classiﬁcation\\n\\n\\x0c',\n",
       " '156\\n\\n18\\n\\nNonlinear least squares\\n\\nFigure 18.6 Least squares and orthogonal distance regression ﬁt of a cubic\\n\\npolynomial to 25 data points.\\n\\n0.20.40.60.81.0-0.2-0.10.00.10.2LSOrth. dist.\\x0c',\n",
       " 'Chapter 19\\n\\nConstrained nonlinear least\\n\\nsquares\\n\\n19.1\\n\\nConstrained nonlinear least squares\\n\\n19.2\\n\\nPenalty algorithm\\n\\nLet’s implement the penalty algorithm (algorithm 19.1 in VMLS).\\n\\n1\\n\\n2\\n\\n3\\n\\n4\\n\\n5\\n\\n6\\n\\n7\\n\\n8\\n\\n9\\n\\n10\\n\\n11\\n\\n12\\n\\n13\\n\\n14\\n\\n15\\n\\n16\\n\\n17\\n\\n18\\n\\n19\\n\\n20\\n\\nfunction penalty_method(f,\\n\\nDf,\\n\\ng,\\n\\nDg,\\n\\nx1,\\n\\nlambda1;\\n\\nkmax\\n\\n=\\n\\n100,\\n\\nfeas_tol\\n\\n=\\n\\n1e-4,\\n\\noc_tol\\n\\n=\\n\\n1e-4)\\n\\nx\\n\\n= x1\\n\\nmu =\\n\\n1.0\\n\\nfeas_res\\n\\noc_res =\\n\\n=\\n[norm(g(x))]\\n[norm(2*Df(x)\\'*f(x) +\\n\\nlm_iters = zeros(Int64,0,1);\\n\\nfor k=1:kmax\\n\\n2*mu*Dg(x)\\'*g(x))]\\n\\nF(x)\\n\\n= [f(x);\\n\\nsqrt(mu)*g(x)]\\n\\nDF(x)\\n\\n= [Df(x);\\n\\nsqrt(mu)*Dg(x)]\\n\\nx, hist =\\n\\nlevenberg_marquardt(F,DF,x,lambda1,tol=oc_tol)\\n\\nfeas_res\\n\\n=\\n\\n[feas_res;\\n\\nnorm(g(x))]\\n\\noc_res =\\n\\n[oc_res;\\n\\nhist[\"residuals\"][end]]\\n\\nlm_iters\\n\\n=\\n\\n[lm_iters;\\n\\nlength(hist[\"residuals\"])]\\n\\nif norm(g(x))\\n\\n<\\n\\nfeas_tol\\n\\nbreak\\n\\nend\\n\\nmu =\\n\\n2*mu\\n\\nend\\n\\nreturn x,\\n\\nDict([\\n\\n(\"lm_iterations\",\\n\\nlm_iters),\\n\\n\\x0c',\n",
       " '158\\n\\n19\\n\\nConstrained nonlinear least squares\\n\\n21\\n\\n(\"feas_res\", feas_res),\\n\\n(\"oc_res\",\\n\\noc_res)\\n\\n])\\n\\n22\\n\\nend\\n\\nOn line 11 we call the function levenberg_marquardt of the previous chapter to\\nminimize (cid:107)F (x)(cid:107)2 where\\n\\nF (x) =\\n\\n(cid:20)\\n\\nf (x)\\n\\n√\\n\\nµg(x)\\n\\n(cid:21)\\n\\n.\\n\\nWe evaluate two residuals. The “feasibility” residual (cid:107)g(x(k) )(cid:107) is the error in the\\n\\nconstraint g(x) = 0. The “optimality condition” residual\\n\\nis deﬁned as\\n\\n(cid:107)2Df (x(k) )T f (x(k) ) + 2Dg(x(k) )T z (k)(cid:107)\\n\\nwhere z (k) = 2µ(k−1) g(x(k) )\\n\\n(and we\\n\\ntake µ(0) = µ(1) ).\\n\\nOn line 13, we obtain\\n\\nthe optimality condition residual as the last residual\\n\\nin the Levenberg–Marquardt\\n\\nmethod. On line 20 we return the ﬁnal x, and a dictionary containing the two se-\\n\\nquences of residuals and the number of iterations used in each call to the Levenberg–\\n\\nMarquardt algorithm.\\n\\n(cid:20) x1 + exp(−x2 )\\nExample. We apply the method to a problem with two variables\\nx2\\n1 + 2x2 + 1\\n\\nf (x1 , x2 ) =\\n\\n(cid:21)\\n\\n,\\n\\ng(x1 , x2 ) = x2\\n1 + x3\\n1 + x2 + x2\\n2 .\\n\\njulia> f(x) =\\n\\n[\\n\\nx[1]\\n\\n+\\n\\nexp(-x[2]),\\n\\nx[1]^2\\n\\n+\\n\\n2*x[2]\\n\\n+\\n\\n1\\n\\n];\\n\\njulia> Df(x) =\\n\\n[\\n\\n1.0\\n\\n-\\n\\nexp(-x[2]);\\n\\n2*x[1]\\n\\n2\\n\\n];\\n\\njulia> g(x) =\\n\\n[\\n\\nx[1]\\n\\n+\\n\\nx[1]^3\\n\\n+\\n\\nx[2]\\n\\n+\\n\\nx[2]^2\\n\\n];\\n\\njulia> Dg(x) =\\n\\n[\\n\\n1\\n\\n+\\n\\n3*x[1]^2\\n\\n1\\n\\n+\\n\\n2*x[2]\\n\\n];\\n\\njulia> x,\\n\\nhist\\n\\n=\\n\\npenalty_method(f,\\n\\nDf,\\n\\ng,\\n\\nDg,\\n\\n[0.5,\\n\\n-0.5],\\n\\n1.0);\\n\\njulia> x\\n\\n2-element\\n\\nArray{Float64,1}:\\n\\n-3.334955140841332e-5\\n\\n-2.7682497163944097e-5\\n\\nThe following lines create a staircase plot with the residuals versus the cumulative\\n\\nnumber of Levenberg–Marquardt iterations as in VMLS ﬁgure 19.4. The result is\\n\\nin ﬁgure 19.1.\\n\\njulia> using Plots\\n\\njulia> cum_lm_iters\\n\\n=\\n\\ncumsum(hist[\"lm_iterations\"],\\n\\ndims=1);\\n\\njulia> itr\\n\\n= vcat([0],\\n\\n[[i;\\n\\ni]\\n\\nfor\\n\\ni\\n\\nin\\n\\ncum_lm_iters]...)\\n\\njulia> feas_res\\n\\n=\\n\\nvcat([\\n\\n[r;r]\\n\\nfor\\n\\nr\\n\\nin\\n\\nhist[\"feas_res\"][1:end-1]]...,\\n\\nhist[\"feas_res\"][end]);\\n\\njulia> oc_res\\n\\n=\\n\\nvcat([\\n\\n\\x0c',\n",
       " '19.3\\n\\nAugmented Lagrangian algorithm\\n\\n159\\n\\nFigure 19.1 Feasibility and optimality condition errors versus the cumulative\\n\\nnumber of Levenberg–Marquardt iterations in the penalty algorithm.\\n\\n[r;r]\\n\\nfor\\n\\nr in\\n\\nhist[\"oc_res\"][1:end-1]]...,\\n\\nhist[\"oc_res\"][end]);\\n\\njulia> plot(itr,\\n\\nfeas_res,\\n\\nshape=:circle,\\n\\nlabel\\n\\n= \"Feasibility\")\\n\\njulia> plot!(itr,\\n\\noc_res,\\n\\nshape=:circle,\\n\\nlabel\\n\\n= \"Opt.\\n\\ncond.\")\\n\\njulia> plot!(yscale\\n\\n=\\n\\n:log10,\\n\\nxlabel =\\n\\n\"Cumulative\\n\\nLevenberg--Marquardt\\n\\niterations\",\\n\\nylabel =\\n\\n\"Residual\")\\n\\n19.3\\n\\nAugmented Lagrangian algorithm\\n\\n1\\n\\n2\\n\\n3\\n\\n4\\n\\n5\\n\\n6\\n\\n7\\n\\n8\\n\\n9\\n\\n10\\n\\nfunction aug_lag_method(f,\\n\\nDf,\\n\\ng,\\n\\nDg,\\n\\nx1,\\n\\nlambda1;\\n\\nkmax\\n\\n=\\n\\n100,\\n\\nfeas_tol\\n\\n=\\n\\n1e-4,\\n\\noc_tol\\n\\n=\\n\\n1e-4)\\n\\nx\\n\\nz\\n\\n= x1\\n\\n= zeros(length(g(x)))\\n\\nmu =\\n\\n1.0\\n\\nfeas_res\\n\\noc_res =\\n\\n=\\n[norm(g(x))]\\n[norm(2*Df(x)\\'*f(x) +\\n\\nlm_iters = zeros(Int64,0,1);\\n\\nfor k=1:kmax\\n\\n2*mu*Dg(x)\\'*z)]\\n\\nF(x)\\n\\n=\\n\\n[f(x);\\n\\nsqrt(mu)*(g(x)\\n\\n+\\n\\nz/(2*mu))]\\n\\n02040608010012010-5 10-4 10-3 10-2 10-1 100 101 Cumulative Levenberg-Marquardt iterationsResidualFeasibilityOpt. cond.\\x0c',\n",
       " '160\\n\\n19\\n\\nConstrained nonlinear least squares\\n\\n11\\n\\nDF(x)\\n\\n= [Df(x);\\n\\nsqrt(mu)*Dg(x)]\\n\\n12\\n\\nx, hist =\\n\\nlevenberg_marquardt(F,\\n\\nDF,\\n\\nx,\\n\\nlambda1,\\n\\ntol=oc_tol)\\n\\n13\\n\\nz\\n\\n= z\\n\\n+ 2*mu*g(x)\\n\\n14\\n\\nfeas_res\\n\\n=\\n\\n[feas_res;\\n\\nnorm(g(x))]\\n\\n15\\n\\noc_res =\\n\\n[oc_res;\\n\\nhist[\"residuals\"][end]]\\n\\n16\\n\\nlm_iters\\n\\n=\\n\\n[lm_iters;\\n\\nlength(hist[\"residuals\"])]\\n\\n17\\n\\nif norm(g(x)) <\\n\\nfeas_tol\\n\\n18\\n\\nbreak\\n\\n19\\n\\nend\\n\\n20\\n\\nmu =\\n\\n(norm(g(x))\\n\\n<\\n\\n0.25*feas_res[end-1])\\n\\n?\\n\\nmu\\n\\n:\\n\\n2*mu\\n\\n21\\n\\nend\\n\\n22\\n\\nreturn x,\\n\\nz,\\n\\nDict([\\n\\n(\"lm_iterations\",\\n\\nlm_iters),\\n\\n23\\n\\n(\"feas_res\", feas_res),\\n\\n(\"oc_res\",\\n\\noc_res)])\\n\\n24\\n\\nend\\n\\nHere the call\\nto the Levenberg–Marquardt algorithm on line 12 is\\n(cid:107)F (x)(cid:107)2 where\\n\\nto minimizes\\n\\nF (x) =\\n\\n(cid:20)\\n\\nf (x)\\n\\n(cid:112)\\n\\nµ(k) (g(x) + z (k)/(2µ(k) ))\\n\\n(cid:21)\\n\\n.\\n\\nWe again record the\\n\\nfeasibility residuals (cid:107)g(x(k)(cid:107) and the optimality conditions\\n\\nresiduals\\n\\n(cid:107)2Df (x(k) )T f (x(k) ) + 2Dg(x(k) )T z (k)(cid:107),\\n\\nand return them in a dictionary.\\n\\nExample. We continue the small example.\\n\\njulia> x,\\n\\nz, hist\\n\\n=\\n\\naug_lag_method(f,\\n\\nDf,\\n\\ng,\\n\\nDg, [0.5,\\n\\n-0.5],\\n\\n1.0);\\n\\njulia> x\\n\\n2-element\\n\\nArray{Float64,1}:\\n\\n-1.8646614856169702e-5\\n\\n-1.5008567819930016e-5\\n\\njulia> z\\n\\n1-element\\n\\nArray{Float64,1}:\\n\\n-1.9999581273499105\\n\\nThe following code shows\\n\\nthe convergence as\\n\\nin VMLS ﬁgure 19.4.\\n\\nThe plot\\n\\nis\\n\\ngiven in ﬁgure 19.2.\\n\\njulia> using Plots\\n\\njulia> cum_lm_iters\\n\\n=\\n\\ncumsum(hist[\"lm_iterations\"],dims=1);\\n\\njulia> itr\\n\\n= vcat([0],\\n\\n[[i;\\n\\ni]\\n\\nfor\\n\\ni\\n\\nin\\n\\ncum_lm_iters]...)\\n\\njulia> feas_res\\n\\n=\\n\\nvcat([\\n\\n[r;r]\\n\\nfor\\n\\nr\\n\\nin\\n\\nhist[\"feas_res\"][1:end-1]]...,\\n\\n\\x0c',\n",
       " '19.4\\n\\nNonlinear control\\n\\n161\\n\\nFigure 19.2 Feasibility and optimality condition errors versus the cumulative\\n\\nnumber of Levenberg–Marquardt\\n\\niterations\\n\\nin the augmented Lagrangian\\n\\nalgorithm.\\n\\nhist[\"feas_res\"][end]);\\n\\njulia> oc_res\\n\\n=\\n\\nvcat([\\n\\n[r;r]\\n\\nfor\\n\\nr in\\n\\nhist[\"oc_res\"][1:end-1]]...,\\n\\nhist[\"oc_res\"][end]);\\n\\njulia> plot(itr,\\n\\nfeas_res,\\n\\nshape=:circle,\\n\\n,\\n\\nlabel =\\n\\n\"Feasibility\")\\n\\njulia> plot!(itr,\\n\\noc_res,\\n\\nshape=:circle,\\n\\nlabel\\n\\n= \"Opt.\\n\\ncond.\")\\n\\njulia> plot!(yscale\\n\\n=\\n\\n:log10,\\n\\nxlabel\\n\\n= \"Cumulative\\n\\nLevenberg-Marquardt\\n\\niterations\",\\n\\nylabel\\n\\n= \"Residual\")\\n\\n19.4\\n\\nNonlinear control\\n\\n025507510010-4 10-3 10-2 10-1 100 101 Cumulative Levenberg-Marquardt iterationsResidualFeasibilityOpt. cond.\\x0c',\n",
       " '162\\n\\n19\\n\\nConstrained nonlinear least squares\\n\\n\\x0c',\n",
       " 'Appendices\\n\\n\\x0c',\n",
       " '\\x0c',\n",
       " 'Appendix A\\n\\nThe VMLS package\\n\\nAfter installing the VMLS package as described on page ix, you can use it by typing\\n\\nusing\\n\\nVMLS at\\n\\nthe Julia prompt. Typing ? followed by a function name gives a\\n\\nshort description of the function.\\n\\nThe VMLS package includes\\n\\nthree types of\\n\\nfunctions:\\n\\nsimple utility functions\\n\\nthat match the VMLS notation or are simpler to use than the corresponding Julia\\n\\nfunctions,\\n\\nimplementations of some algorithms in VMLS, and functions that gen-\\n\\nerate data for\\n\\nsome of\\n\\nthe examples. The algorithm implementations are meant\\n\\nfor use in the examples of\\n\\nthis companion and for exercises\\n\\nin VMLS. They are\\n\\nnot optimized for eﬃciency or robustness, and do not perform any error checking\\n\\nof the input arguments.\\n\\nA.1\\n\\nUtility functions\\n\\nVector utility functions.\\n\\navg(x). Returns the average of the elements of a vector or matrix (page 20).\\n\\nrms(x). Returns the RMS value of the elements of a vector or matrix (page 25).\\n\\nstdev(x). Returns the standard deviation of the elements of a vector or matrix\\n\\n(page 28).\\n\\nang(x,y). Returns the angle in radians between non-zero vectors (page 29).\\n\\ncorrel_coeff(x,y).\\n\\nReturns\\n\\nthe\\n\\ncorrelation coeﬃcient between non-constant\\n\\nvectors (page 30).\\n\\nMatrix utility functions.\\n\\neye(n). Returns an n × n identity matrix (page 50).\\n\\ndiagonal(x). Returns a diagonal matrix with the entries of the vector x on its\\n\\ndiagonal (page 52).\\n\\n\\x0c',\n",
       " '166\\n\\nA\\n\\nThe VMLS package\\n\\nspeye(n). Returns an n × n sparse identity matrix (page 54).\\n\\nspdiagonal(x). Returns a sparse diagonal matrix with the entries of the vector\\n\\nx on its diagonal (page 54).\\n\\nvandermonde(t,n). Returns\\n\\ncolumn ti−1 (page 58).\\n\\nthe Vandermonde matrix with n columns and ith\\n\\ntoeplitz(a,n). Returns the Toeplitz matrix with n columns and the vector a in\\n\\nthe leading positions of the ﬁrst column (page 65).\\n\\nRange function.\\n\\nlinspace(a,b,n). Returns a vector with n equally spaced numbers between a\\n\\nand b (page 90).\\n\\nUtility functions for classiﬁcation.\\n\\nconfusion_matrix(y,yhat,K=2). Returns the confusion matrix for a data vector\\n\\ny and the vector of predictions ˆy .\\nIf K = 2, the vectors y and ˆy are Boolean.\\nIf K > 2, they contain integers in {1, . . . , K } (pages 113, 115).\\n\\nrow_argmax(X).\\n\\nIf X is an m × n matrix, returns an m-vector with ith element\\n\\nargmaxj Xij\\n\\n(page 116).\\n\\none_hot(x,K). Returns\\n\\nthe one-hot encoding of\\nthe vector x, which must have\\nelements in {1, . . . , K }. The one-hot encoding of an n-vector x is the n × K\\n\\nmatrix X with Xij = 1 if xi = j and Xij = 0 otherwise (page 117).\\n\\nA.2\\n\\nAlgorithms\\n\\nk_means(X,k;maxiters=100,tol=1e-5).\\n\\nApplies\\n\\nthe k-means algorithm for k\\n\\nclusters\\nto the vectors\\nstored in X. The argument X is a one-dimensional\\narray of N n-vectors, or an n × N -matrix.\\n\\nfunction returns a tuple\\n\\nThe\\n\\nwith two elements. The ﬁrst output argument\\n{1, . . . , k} with the cluster assignments\\n\\nfor\\n\\nis an array of N integers in\\n\\nthe N data points. The second\\n\\noutput argument is an array of k n-vectors with the k cluster representatives\\n\\n(page 34).\\n\\ngram_schmidt(a;tol=1e-10). Applies the Gram–Schmidt algorithm to the vec-\\n\\ntor stored in the array a and returns the result as an array of vectors (page 41).\\n\\nmols_solve(As,Bs,lambdas). Returns the solution of the multi-ob jective least\\n\\nsquares problem with coeﬃcient matrices\\n\\nin the array As,\\n\\nright-hand side\\n\\nvectors in the array bs, and weights in the array lambdas (page 121).\\n\\ncls_solve(A,b,C,d). Returns the solution of the constrained least squares prob-\\n\\nlem with coeﬃcient matrices A and C , and right-hand side vectors or matrices\\n\\nb and d (page 132).\\n\\n\\x0c',\n",
       " 'A.3\\n\\nData sets\\n\\n167\\n\\nlevenberg_marquardt(f,Df,x1,lambda1;kmax=100,tol=1e-6). Applies the Levenberg–\\n\\nMarquardt algorithm to the function deﬁned in f and Df, with starting point\\nx(1) and initial regularization parameter λ(1) . The function returns the ﬁnal\\n\\niterate x and a dictionary with the convergence history (page 147).\\n\\naug_lag_method(f,Df,g,Dg,x1,lambda1;kmax=100,feas_tol=1e-4,oc_tol=1e-4).\\n\\nApplies the augmented Lagrangian method to the constrained nonlinear least\\nsquares problem deﬁned by f, Df, g, Dg, with starting point x(1) . The sub-\\n\\nproblems are solved using the Levenberg–Marquardt method with initial reg-\\nularization parameter λ(1) . Returns\\n\\niterate x, multiplier z , and a\\n\\nthe ﬁnal\\n\\ndictionary with the convergence history (page 159).\\n\\nA.3\\n\\nData sets\\n\\nhouse_sales_data(). Returns a dictionary D with the Sacramento house sales\\n\\ndata used in section 2.3 and chapter 13 of VMLS. The 6 items in the dictionary\\n\\nare vectors of\\n\\nlength 774, with data for 774 house sales.\\n\\nD[\"price\"]:\\n\\nD[\"area\"]:\\n\\nD[\"beds\"]:\\n\\nD[\"baths\"]:\\n\\nD[\"condo\"]:\\n\\nselling price in 1000 dollars\\n\\narea in 1000 square feet\\n\\nnumber of bedrooms\\n\\nnumber of bathrooms\\n\\n1 if a condo, 0 otherwise\\n\\nD[\"location\"]:\\n\\nan integer between 1 and 4 indicating the location.\\n\\npopulation_data(). Returns a dictionary D with the US population data used in\\n\\nsection 9.2 of VMLS. The items in the dictionary are three vectors of\\n\\nlength\\n\\n100.\\n\\nD[\"population\"]:\\n\\n2010 population in millions for ages 0,. . . ,99\\n\\nD[\"birth_rate\"]:\\n\\nbirth rate\\n\\nD[\"death_rate\"]:\\n\\ndeath rate.\\n\\npetroleum_consumption_data().\\n\\nReturns a 34-vector with the world annual\\n\\npetroleum consumption between 1980 and 2013, in thousand barrels/day (dis-\\n\\ncussed on page 252 in VMLS).\\n\\nvehicle_miles_data(). Returns a 15 matrix with the vehicle miles traveled in\\n\\nthe US (in millions), per month,\\n\\nfor\\n\\nthe years 2000, . . . , 2014 (discussed on\\n\\npage 252 in VMLS).\\n\\ntemperature_data(). Returns a vector of\\n\\nlength 774 = 31 · 24 with the hourly\\n\\ntemperature at LAX in May 2016 (discussed on pages 259 and 266 in VMLS).\\n\\niris_data(). Returns a dictionary D with the Iris ﬂower data set, discussed in\\n\\nsections 14.2.1 and 14.3.2 of VMLS. The items in the dictionary are:\\n\\n\\x0c',\n",
       " '168\\n\\nA\\n\\nThe VMLS package\\n\\nD[\"setosa\"]\\n\\nD[\"versicolor\"]:\\n\\nD[\"virginica\"]:\\n\\na 50 × 4 matrix with 50 examples of Iris Setosa\\na 50 × 4 matrix with 50 examples of Iris Versicolor\\na 50 × 4 matrix with 50 examples of Iris Virginica.\\n\\nThe columns give values for four features:\\n\\nsepal\\n\\nlength in cm, sepal width in\\n\\ncm, petal\\n\\nlength in cm, petal width in cm.\\n\\nozone_data(). Returns a vector of\\n\\nlength 336 = 14 · 24 with the hourly ozone\\n\\nlevels at Azusa, California, during the ﬁrst 14 days of July 2014 (discussed\\n\\non page 319 in VMLS).\\n\\nregularized_fit_data(). Returns a dictionary D with data for the regularized\\n\\ndata ﬁtting example on page 329 of VMLS. The items in the dictionary are:\\n\\nD[\"xtrain\"]:\\n\\nD[\"ytrain\"]:\\n\\nD[\"xtest\"]:\\n\\nD[\"ytest\"]:\\n\\nvector of\\n\\nlength 10\\n\\nvector of\\n\\nlength 10\\n\\nvector of\\n\\nlength 20\\n\\nvector of\\n\\nlength 20.\\n\\nportfolio_data(). Returns a tuple (R,\\n\\nRtest) with data for the portfolio op-\\ntimization example in section 17.1.3 of VMLS. R is a 2000 × 20 matrix with\\ndaily returns over a period of 2000 days. The ﬁrst 19 columns are returns for\\n19 stocks; the last column is for a risk-free asset. Rtest is a 500 × 20 matrix\\nwith daily returns over a diﬀerent period of 500 days.\\n\\nlq_estimation_data(). Returns a 2 × 100 matrix with the measurement data\\nfor the linear quadratic state estimation example of section 17.3.1 of VMLS.\\n\\north_dist_reg_data() Returns a tuple (xd, yd) with the data for the orthog-\\n\\nonal distance regression example on page 400 in VMLS.\\n\\n\\x0c']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extracted_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting tika\n",
      "  Downloading https://files.pythonhosted.org/packages/0e/0f/ddb78a3220c07be34e7cf8105daee7080eb0947796498c6f2a01284b0c29/tika-1.22.tar.gz\n",
      "Requirement already satisfied: setuptools in /Users/vb/anaconda3/lib/python3.7/site-packages (from tika) (41.0.1)\n",
      "Requirement already satisfied: requests in /Users/vb/anaconda3/lib/python3.7/site-packages (from tika) (2.22.0)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /Users/vb/anaconda3/lib/python3.7/site-packages (from requests->tika) (1.24.2)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /Users/vb/anaconda3/lib/python3.7/site-packages (from requests->tika) (2.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/vb/anaconda3/lib/python3.7/site-packages (from requests->tika) (2019.6.16)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /Users/vb/anaconda3/lib/python3.7/site-packages (from requests->tika) (3.0.4)\n",
      "Building wheels for collected packages: tika\n",
      "  Building wheel for tika (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for tika: filename=tika-1.22-cp37-none-any.whl size=30410 sha256=7217d38e7de2c2d54d5d8aac66327c9d8517d00ac63982194489adc5f5630c50\n",
      "  Stored in directory: /Users/vb/Library/Caches/pip/wheels/76/9d/c5/be9d60fca685060535e2e9635a42f77abc0d520c0eed98fcfd\n",
      "Successfully built tika\n",
      "Installing collected packages: tika\n",
      "Successfully installed tika-1.22\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install tika"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tika import parser\n",
    "\n",
    "rawText = parser.from_file('indpages/companion_page_1.pdf')\n",
    "\n",
    "# rawList = rawText['content'].splitlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "str"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(rawText['content'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " 'Contents',\n",
       " '',\n",
       " 'Preface vii',\n",
       " '',\n",
       " 'Getting started with Julia ix',\n",
       " '',\n",
       " '1 Vectors 1',\n",
       " '1.1 Vectors . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 1',\n",
       " '1.2 Vector addition . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 9',\n",
       " '1.3 Scalar-vector multiplication . . . . . . . . . . . . . . . . . . . . . . . . 9',\n",
       " '1.4 Inner product . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 14',\n",
       " '1.5 Complexity of vector computations . . . . . . . . . . . . . . . . . . . . 15',\n",
       " '',\n",
       " '2 Linear functions 19',\n",
       " '2.1 Linear functions . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 19',\n",
       " '2.2 Taylor approximation . . . . . . . . . . . . . . . . . . . . . . . . . . . 20',\n",
       " '2.3 Regression model . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 21',\n",
       " '',\n",
       " '3 Norm and distance 25',\n",
       " '3.1 Norm . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 25',\n",
       " '3.2 Distance . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 27',\n",
       " '3.3 Standard deviation . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 28',\n",
       " '3.4 Angle . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 29',\n",
       " '3.5 Complexity . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 30',\n",
       " '',\n",
       " '4 Clustering 33',\n",
       " '4.1 Clustering . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 33',\n",
       " '4.2 A clustering objective . . . . . . . . . . . . . . . . . . . . . . . . . . . 33',\n",
       " '4.3 The k-means algorithm . . . . . . . . . . . . . . . . . . . . . . . . . . 34',\n",
       " '4.4 Examples . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 37',\n",
       " '4.5 Applications . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 37',\n",
       " '',\n",
       " '5 Linear independence 39',\n",
       " '5.1 Linear dependence . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 39',\n",
       " '5.2 Basis . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 39',\n",
       " '5.3 Orthonormal vectors . . . . . . . . . . . . . . . . . . . . . . . . . . . . 40',\n",
       " '5.4 Gram–Schmidt algorithm . . . . . . . . . . . . . . . . . . . . . . . . . 41',\n",
       " '',\n",
       " '']"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rawList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    " \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pdf_splitter(path):\n",
    "    fname = os.path.splitext(os.path.basename(path))[0]\n",
    " \n",
    "    pdf = pypdf.PdfFileReader(path)\n",
    "    for page in range(pdf.getNumPages()):\n",
    "        pdf_writer = pypdf.PdfFileWriter()\n",
    "        pdf_writer.addPage(pdf.getPage(page))\n",
    " \n",
    "        output_filename = '{}_page_{}.pdf'.format(\n",
    "            fname, page+1)\n",
    " \n",
    "        with open(\"indpages/\"+output_filename, 'wb') as out:\n",
    "            pdf_writer.write(out)\n",
    " \n",
    "        print('Created: {}'.format(output_filename))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created: companion_page_1.pdf\n",
      "Created: companion_page_2.pdf\n",
      "Created: companion_page_3.pdf\n",
      "Created: companion_page_4.pdf\n",
      "Created: companion_page_5.pdf\n",
      "Created: companion_page_6.pdf\n",
      "Created: companion_page_7.pdf\n",
      "Created: companion_page_8.pdf\n",
      "Created: companion_page_9.pdf\n",
      "Created: companion_page_10.pdf\n",
      "Created: companion_page_11.pdf\n",
      "Created: companion_page_12.pdf\n",
      "Created: companion_page_13.pdf\n",
      "Created: companion_page_14.pdf\n",
      "Created: companion_page_15.pdf\n",
      "Created: companion_page_16.pdf\n",
      "Created: companion_page_17.pdf\n",
      "Created: companion_page_18.pdf\n",
      "Created: companion_page_19.pdf\n",
      "Created: companion_page_20.pdf\n",
      "Created: companion_page_21.pdf\n",
      "Created: companion_page_22.pdf\n",
      "Created: companion_page_23.pdf\n",
      "Created: companion_page_24.pdf\n",
      "Created: companion_page_25.pdf\n",
      "Created: companion_page_26.pdf\n",
      "Created: companion_page_27.pdf\n",
      "Created: companion_page_28.pdf\n",
      "Created: companion_page_29.pdf\n",
      "Created: companion_page_30.pdf\n",
      "Created: companion_page_31.pdf\n",
      "Created: companion_page_32.pdf\n",
      "Created: companion_page_33.pdf\n",
      "Created: companion_page_34.pdf\n",
      "Created: companion_page_35.pdf\n",
      "Created: companion_page_36.pdf\n",
      "Created: companion_page_37.pdf\n",
      "Created: companion_page_38.pdf\n",
      "Created: companion_page_39.pdf\n",
      "Created: companion_page_40.pdf\n",
      "Created: companion_page_41.pdf\n",
      "Created: companion_page_42.pdf\n",
      "Created: companion_page_43.pdf\n",
      "Created: companion_page_44.pdf\n",
      "Created: companion_page_45.pdf\n",
      "Created: companion_page_46.pdf\n",
      "Created: companion_page_47.pdf\n",
      "Created: companion_page_48.pdf\n",
      "Created: companion_page_49.pdf\n",
      "Created: companion_page_50.pdf\n",
      "Created: companion_page_51.pdf\n",
      "Created: companion_page_52.pdf\n",
      "Created: companion_page_53.pdf\n",
      "Created: companion_page_54.pdf\n",
      "Created: companion_page_55.pdf\n",
      "Created: companion_page_56.pdf\n",
      "Created: companion_page_57.pdf\n",
      "Created: companion_page_58.pdf\n",
      "Created: companion_page_59.pdf\n",
      "Created: companion_page_60.pdf\n",
      "Created: companion_page_61.pdf\n",
      "Created: companion_page_62.pdf\n",
      "Created: companion_page_63.pdf\n",
      "Created: companion_page_64.pdf\n",
      "Created: companion_page_65.pdf\n",
      "Created: companion_page_66.pdf\n",
      "Created: companion_page_67.pdf\n",
      "Created: companion_page_68.pdf\n",
      "Created: companion_page_69.pdf\n",
      "Created: companion_page_70.pdf\n",
      "Created: companion_page_71.pdf\n",
      "Created: companion_page_72.pdf\n",
      "Created: companion_page_73.pdf\n",
      "Created: companion_page_74.pdf\n",
      "Created: companion_page_75.pdf\n",
      "Created: companion_page_76.pdf\n",
      "Created: companion_page_77.pdf\n",
      "Created: companion_page_78.pdf\n",
      "Created: companion_page_79.pdf\n",
      "Created: companion_page_80.pdf\n",
      "Created: companion_page_81.pdf\n",
      "Created: companion_page_82.pdf\n",
      "Created: companion_page_83.pdf\n",
      "Created: companion_page_84.pdf\n",
      "Created: companion_page_85.pdf\n",
      "Created: companion_page_86.pdf\n",
      "Created: companion_page_87.pdf\n",
      "Created: companion_page_88.pdf\n",
      "Created: companion_page_89.pdf\n",
      "Created: companion_page_90.pdf\n",
      "Created: companion_page_91.pdf\n",
      "Created: companion_page_92.pdf\n",
      "Created: companion_page_93.pdf\n",
      "Created: companion_page_94.pdf\n",
      "Created: companion_page_95.pdf\n",
      "Created: companion_page_96.pdf\n",
      "Created: companion_page_97.pdf\n",
      "Created: companion_page_98.pdf\n",
      "Created: companion_page_99.pdf\n",
      "Created: companion_page_100.pdf\n",
      "Created: companion_page_101.pdf\n",
      "Created: companion_page_102.pdf\n",
      "Created: companion_page_103.pdf\n",
      "Created: companion_page_104.pdf\n",
      "Created: companion_page_105.pdf\n",
      "Created: companion_page_106.pdf\n",
      "Created: companion_page_107.pdf\n",
      "Created: companion_page_108.pdf\n",
      "Created: companion_page_109.pdf\n",
      "Created: companion_page_110.pdf\n",
      "Created: companion_page_111.pdf\n",
      "Created: companion_page_112.pdf\n",
      "Created: companion_page_113.pdf\n",
      "Created: companion_page_114.pdf\n",
      "Created: companion_page_115.pdf\n",
      "Created: companion_page_116.pdf\n",
      "Created: companion_page_117.pdf\n",
      "Created: companion_page_118.pdf\n",
      "Created: companion_page_119.pdf\n",
      "Created: companion_page_120.pdf\n",
      "Created: companion_page_121.pdf\n",
      "Created: companion_page_122.pdf\n",
      "Created: companion_page_123.pdf\n",
      "Created: companion_page_124.pdf\n",
      "Created: companion_page_125.pdf\n",
      "Created: companion_page_126.pdf\n",
      "Created: companion_page_127.pdf\n",
      "Created: companion_page_128.pdf\n",
      "Created: companion_page_129.pdf\n",
      "Created: companion_page_130.pdf\n",
      "Created: companion_page_131.pdf\n",
      "Created: companion_page_132.pdf\n",
      "Created: companion_page_133.pdf\n",
      "Created: companion_page_134.pdf\n",
      "Created: companion_page_135.pdf\n",
      "Created: companion_page_136.pdf\n",
      "Created: companion_page_137.pdf\n",
      "Created: companion_page_138.pdf\n",
      "Created: companion_page_139.pdf\n",
      "Created: companion_page_140.pdf\n",
      "Created: companion_page_141.pdf\n",
      "Created: companion_page_142.pdf\n",
      "Created: companion_page_143.pdf\n",
      "Created: companion_page_144.pdf\n",
      "Created: companion_page_145.pdf\n",
      "Created: companion_page_146.pdf\n",
      "Created: companion_page_147.pdf\n",
      "Created: companion_page_148.pdf\n",
      "Created: companion_page_149.pdf\n",
      "Created: companion_page_150.pdf\n",
      "Created: companion_page_151.pdf\n",
      "Created: companion_page_152.pdf\n",
      "Created: companion_page_153.pdf\n",
      "Created: companion_page_154.pdf\n",
      "Created: companion_page_155.pdf\n",
      "Created: companion_page_156.pdf\n",
      "Created: companion_page_157.pdf\n",
      "Created: companion_page_158.pdf\n",
      "Created: companion_page_159.pdf\n",
      "Created: companion_page_160.pdf\n",
      "Created: companion_page_161.pdf\n",
      "Created: companion_page_162.pdf\n",
      "Created: companion_page_163.pdf\n",
      "Created: companion_page_164.pdf\n",
      "Created: companion_page_165.pdf\n",
      "Created: companion_page_166.pdf\n",
      "Created: companion_page_167.pdf\n",
      "Created: companion_page_168.pdf\n",
      "Created: companion_page_169.pdf\n",
      "Created: companion_page_170.pdf\n",
      "Created: companion_page_171.pdf\n",
      "Created: companion_page_172.pdf\n",
      "Created: companion_page_173.pdf\n",
      "Created: companion_page_174.pdf\n",
      "Created: companion_page_175.pdf\n",
      "Created: companion_page_176.pdf\n",
      "Created: companion_page_177.pdf\n",
      "Created: companion_page_178.pdf\n"
     ]
    }
   ],
   "source": [
    "path = 'companion.pdf'\n",
    "pdf_splitter(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rawText = parser.from_file('companion.pdf')\n",
    "\n",
    "# rawList = rawText['content'].splitlines()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['indpages/companion_page_1.pdf',\n",
       " 'indpages/companion_page_2.pdf',\n",
       " 'indpages/companion_page_3.pdf',\n",
       " 'indpages/companion_page_4.pdf',\n",
       " 'indpages/companion_page_5.pdf',\n",
       " 'indpages/companion_page_6.pdf',\n",
       " 'indpages/companion_page_7.pdf',\n",
       " 'indpages/companion_page_8.pdf',\n",
       " 'indpages/companion_page_9.pdf',\n",
       " 'indpages/companion_page_10.pdf',\n",
       " 'indpages/companion_page_11.pdf',\n",
       " 'indpages/companion_page_12.pdf',\n",
       " 'indpages/companion_page_13.pdf',\n",
       " 'indpages/companion_page_14.pdf',\n",
       " 'indpages/companion_page_15.pdf',\n",
       " 'indpages/companion_page_16.pdf',\n",
       " 'indpages/companion_page_17.pdf',\n",
       " 'indpages/companion_page_18.pdf',\n",
       " 'indpages/companion_page_19.pdf',\n",
       " 'indpages/companion_page_20.pdf',\n",
       " 'indpages/companion_page_21.pdf',\n",
       " 'indpages/companion_page_22.pdf',\n",
       " 'indpages/companion_page_23.pdf',\n",
       " 'indpages/companion_page_24.pdf',\n",
       " 'indpages/companion_page_25.pdf',\n",
       " 'indpages/companion_page_26.pdf',\n",
       " 'indpages/companion_page_27.pdf',\n",
       " 'indpages/companion_page_28.pdf',\n",
       " 'indpages/companion_page_29.pdf',\n",
       " 'indpages/companion_page_30.pdf',\n",
       " 'indpages/companion_page_31.pdf',\n",
       " 'indpages/companion_page_32.pdf',\n",
       " 'indpages/companion_page_33.pdf',\n",
       " 'indpages/companion_page_34.pdf',\n",
       " 'indpages/companion_page_35.pdf',\n",
       " 'indpages/companion_page_36.pdf',\n",
       " 'indpages/companion_page_37.pdf',\n",
       " 'indpages/companion_page_38.pdf',\n",
       " 'indpages/companion_page_39.pdf',\n",
       " 'indpages/companion_page_40.pdf',\n",
       " 'indpages/companion_page_41.pdf',\n",
       " 'indpages/companion_page_42.pdf',\n",
       " 'indpages/companion_page_43.pdf',\n",
       " 'indpages/companion_page_44.pdf',\n",
       " 'indpages/companion_page_45.pdf',\n",
       " 'indpages/companion_page_46.pdf',\n",
       " 'indpages/companion_page_47.pdf',\n",
       " 'indpages/companion_page_48.pdf',\n",
       " 'indpages/companion_page_49.pdf',\n",
       " 'indpages/companion_page_50.pdf',\n",
       " 'indpages/companion_page_51.pdf',\n",
       " 'indpages/companion_page_52.pdf',\n",
       " 'indpages/companion_page_53.pdf',\n",
       " 'indpages/companion_page_54.pdf',\n",
       " 'indpages/companion_page_55.pdf',\n",
       " 'indpages/companion_page_56.pdf',\n",
       " 'indpages/companion_page_57.pdf',\n",
       " 'indpages/companion_page_58.pdf',\n",
       " 'indpages/companion_page_59.pdf',\n",
       " 'indpages/companion_page_60.pdf',\n",
       " 'indpages/companion_page_61.pdf',\n",
       " 'indpages/companion_page_62.pdf',\n",
       " 'indpages/companion_page_63.pdf',\n",
       " 'indpages/companion_page_64.pdf',\n",
       " 'indpages/companion_page_65.pdf',\n",
       " 'indpages/companion_page_66.pdf',\n",
       " 'indpages/companion_page_67.pdf',\n",
       " 'indpages/companion_page_68.pdf',\n",
       " 'indpages/companion_page_69.pdf',\n",
       " 'indpages/companion_page_70.pdf',\n",
       " 'indpages/companion_page_71.pdf',\n",
       " 'indpages/companion_page_72.pdf',\n",
       " 'indpages/companion_page_73.pdf',\n",
       " 'indpages/companion_page_74.pdf',\n",
       " 'indpages/companion_page_75.pdf',\n",
       " 'indpages/companion_page_76.pdf',\n",
       " 'indpages/companion_page_77.pdf',\n",
       " 'indpages/companion_page_78.pdf',\n",
       " 'indpages/companion_page_79.pdf',\n",
       " 'indpages/companion_page_80.pdf',\n",
       " 'indpages/companion_page_81.pdf',\n",
       " 'indpages/companion_page_82.pdf',\n",
       " 'indpages/companion_page_83.pdf',\n",
       " 'indpages/companion_page_84.pdf',\n",
       " 'indpages/companion_page_85.pdf',\n",
       " 'indpages/companion_page_86.pdf',\n",
       " 'indpages/companion_page_87.pdf',\n",
       " 'indpages/companion_page_88.pdf',\n",
       " 'indpages/companion_page_89.pdf',\n",
       " 'indpages/companion_page_90.pdf',\n",
       " 'indpages/companion_page_91.pdf',\n",
       " 'indpages/companion_page_92.pdf',\n",
       " 'indpages/companion_page_93.pdf',\n",
       " 'indpages/companion_page_94.pdf',\n",
       " 'indpages/companion_page_95.pdf',\n",
       " 'indpages/companion_page_96.pdf',\n",
       " 'indpages/companion_page_97.pdf',\n",
       " 'indpages/companion_page_98.pdf',\n",
       " 'indpages/companion_page_99.pdf',\n",
       " 'indpages/companion_page_100.pdf',\n",
       " 'indpages/companion_page_101.pdf',\n",
       " 'indpages/companion_page_102.pdf',\n",
       " 'indpages/companion_page_103.pdf',\n",
       " 'indpages/companion_page_104.pdf',\n",
       " 'indpages/companion_page_105.pdf',\n",
       " 'indpages/companion_page_106.pdf',\n",
       " 'indpages/companion_page_107.pdf',\n",
       " 'indpages/companion_page_108.pdf',\n",
       " 'indpages/companion_page_109.pdf',\n",
       " 'indpages/companion_page_110.pdf',\n",
       " 'indpages/companion_page_111.pdf',\n",
       " 'indpages/companion_page_112.pdf',\n",
       " 'indpages/companion_page_113.pdf',\n",
       " 'indpages/companion_page_114.pdf',\n",
       " 'indpages/companion_page_115.pdf',\n",
       " 'indpages/companion_page_116.pdf',\n",
       " 'indpages/companion_page_117.pdf',\n",
       " 'indpages/companion_page_118.pdf',\n",
       " 'indpages/companion_page_119.pdf',\n",
       " 'indpages/companion_page_120.pdf',\n",
       " 'indpages/companion_page_121.pdf',\n",
       " 'indpages/companion_page_122.pdf',\n",
       " 'indpages/companion_page_123.pdf',\n",
       " 'indpages/companion_page_124.pdf',\n",
       " 'indpages/companion_page_125.pdf',\n",
       " 'indpages/companion_page_126.pdf',\n",
       " 'indpages/companion_page_127.pdf',\n",
       " 'indpages/companion_page_128.pdf',\n",
       " 'indpages/companion_page_129.pdf',\n",
       " 'indpages/companion_page_130.pdf',\n",
       " 'indpages/companion_page_131.pdf',\n",
       " 'indpages/companion_page_132.pdf',\n",
       " 'indpages/companion_page_133.pdf',\n",
       " 'indpages/companion_page_134.pdf',\n",
       " 'indpages/companion_page_135.pdf',\n",
       " 'indpages/companion_page_136.pdf',\n",
       " 'indpages/companion_page_137.pdf',\n",
       " 'indpages/companion_page_138.pdf',\n",
       " 'indpages/companion_page_139.pdf',\n",
       " 'indpages/companion_page_140.pdf',\n",
       " 'indpages/companion_page_141.pdf',\n",
       " 'indpages/companion_page_142.pdf',\n",
       " 'indpages/companion_page_143.pdf',\n",
       " 'indpages/companion_page_144.pdf',\n",
       " 'indpages/companion_page_145.pdf',\n",
       " 'indpages/companion_page_146.pdf',\n",
       " 'indpages/companion_page_147.pdf',\n",
       " 'indpages/companion_page_148.pdf',\n",
       " 'indpages/companion_page_149.pdf',\n",
       " 'indpages/companion_page_150.pdf',\n",
       " 'indpages/companion_page_151.pdf',\n",
       " 'indpages/companion_page_152.pdf',\n",
       " 'indpages/companion_page_153.pdf',\n",
       " 'indpages/companion_page_154.pdf',\n",
       " 'indpages/companion_page_155.pdf',\n",
       " 'indpages/companion_page_156.pdf',\n",
       " 'indpages/companion_page_157.pdf',\n",
       " 'indpages/companion_page_158.pdf',\n",
       " 'indpages/companion_page_159.pdf',\n",
       " 'indpages/companion_page_160.pdf',\n",
       " 'indpages/companion_page_161.pdf',\n",
       " 'indpages/companion_page_162.pdf',\n",
       " 'indpages/companion_page_163.pdf',\n",
       " 'indpages/companion_page_164.pdf',\n",
       " 'indpages/companion_page_165.pdf',\n",
       " 'indpages/companion_page_166.pdf',\n",
       " 'indpages/companion_page_167.pdf',\n",
       " 'indpages/companion_page_168.pdf',\n",
       " 'indpages/companion_page_169.pdf',\n",
       " 'indpages/companion_page_170.pdf',\n",
       " 'indpages/companion_page_171.pdf',\n",
       " 'indpages/companion_page_172.pdf',\n",
       " 'indpages/companion_page_173.pdf',\n",
       " 'indpages/companion_page_174.pdf',\n",
       " 'indpages/companion_page_175.pdf',\n",
       " 'indpages/companion_page_176.pdf',\n",
       " 'indpages/companion_page_177.pdf',\n",
       " 'indpages/companion_page_178.pdf']"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paths = [\"indpages/companion_page_\"+str(i)+\".pdf\" for i in range(1,179)]\n",
    "paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "def construct_list(paths):\n",
    "    outList = []\n",
    "    for i in paths:\n",
    "        rawText = parser.from_file(i)\n",
    "        if type(rawText['content']) == str:\n",
    "            rawList = rawText['content'].splitlines()\n",
    "            outList.append(rawList)\n",
    "        else:\n",
    "            outList.append(\"Empty\")\n",
    "    return outList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "megalist = construct_list(paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Introduction to Applied Linear Algebra',\n",
       "  '',\n",
       "  'Vectors, Matrices, and Least Squares',\n",
       "  '',\n",
       "  'Julia Language Companion',\n",
       "  '',\n",
       "  'Stephen Boyd and Lieven Vandenberghe',\n",
       "  '',\n",
       "  'DRAFT August 26, 2018',\n",
       "  '',\n",
       "  ''],\n",
       " 'Empty',\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Contents',\n",
       "  '',\n",
       "  'Preface vii',\n",
       "  '',\n",
       "  'Getting started with Julia ix',\n",
       "  '',\n",
       "  '1 Vectors 1',\n",
       "  '1.1 Vectors . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 1',\n",
       "  '1.2 Vector addition . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 9',\n",
       "  '1.3 Scalar-vector multiplication . . . . . . . . . . . . . . . . . . . . . . . . 9',\n",
       "  '1.4 Inner product . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 14',\n",
       "  '1.5 Complexity of vector computations . . . . . . . . . . . . . . . . . . . . 15',\n",
       "  '',\n",
       "  '2 Linear functions 19',\n",
       "  '2.1 Linear functions . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 19',\n",
       "  '2.2 Taylor approximation . . . . . . . . . . . . . . . . . . . . . . . . . . . 20',\n",
       "  '2.3 Regression model . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 21',\n",
       "  '',\n",
       "  '3 Norm and distance 25',\n",
       "  '3.1 Norm . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 25',\n",
       "  '3.2 Distance . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 27',\n",
       "  '3.3 Standard deviation . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 28',\n",
       "  '3.4 Angle . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 29',\n",
       "  '3.5 Complexity . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 30',\n",
       "  '',\n",
       "  '4 Clustering 33',\n",
       "  '4.1 Clustering . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 33',\n",
       "  '4.2 A clustering objective . . . . . . . . . . . . . . . . . . . . . . . . . . . 33',\n",
       "  '4.3 The k-means algorithm . . . . . . . . . . . . . . . . . . . . . . . . . . 34',\n",
       "  '4.4 Examples . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 37',\n",
       "  '4.5 Applications . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 37',\n",
       "  '',\n",
       "  '5 Linear independence 39',\n",
       "  '5.1 Linear dependence . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 39',\n",
       "  '5.2 Basis . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 39',\n",
       "  '5.3 Orthonormal vectors . . . . . . . . . . . . . . . . . . . . . . . . . . . . 40',\n",
       "  '5.4 Gram–Schmidt algorithm . . . . . . . . . . . . . . . . . . . . . . . . . 41',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'iv Contents',\n",
       "  '',\n",
       "  '6 Matrices 45',\n",
       "  '6.1 Matrices . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 45',\n",
       "  '6.2 Zero and identity matrices . . . . . . . . . . . . . . . . . . . . . . . . 50',\n",
       "  '6.3 Transpose, addition, and norm . . . . . . . . . . . . . . . . . . . . . . 54',\n",
       "  '6.4 Matrix-vector multiplication . . . . . . . . . . . . . . . . . . . . . . . . 56',\n",
       "  '6.5 Complexity . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 59',\n",
       "  '',\n",
       "  '7 Matrix examples 61',\n",
       "  '7.1 Geometric transformations . . . . . . . . . . . . . . . . . . . . . . . . 61',\n",
       "  '7.2 Selectors . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 61',\n",
       "  '7.3 Incidence matrix . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 63',\n",
       "  '7.4 Convolution . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 65',\n",
       "  '',\n",
       "  '8 Linear equations 67',\n",
       "  '8.1 Linear and affine functions . . . . . . . . . . . . . . . . . . . . . . . . 67',\n",
       "  '8.2 Linear function models . . . . . . . . . . . . . . . . . . . . . . . . . . 69',\n",
       "  '8.3 Systems of linear equations . . . . . . . . . . . . . . . . . . . . . . . . 71',\n",
       "  '',\n",
       "  '9 Linear dynamical systems 73',\n",
       "  '9.1 Linear dynamical systems . . . . . . . . . . . . . . . . . . . . . . . . . 73',\n",
       "  '9.2 Population dynamics . . . . . . . . . . . . . . . . . . . . . . . . . . . 74',\n",
       "  '9.3 Epidemic dynamics . . . . . . . . . . . . . . . . . . . . . . . . . . . . 75',\n",
       "  '9.4 Motion of a mass . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 75',\n",
       "  '9.5 Supply chain dynamics . . . . . . . . . . . . . . . . . . . . . . . . . . 76',\n",
       "  '',\n",
       "  '10 Matrix multiplication 79',\n",
       "  '10.1 Matrix-matrix multiplication . . . . . . . . . . . . . . . . . . . . . . . 79',\n",
       "  '10.2 Composition of linear functions . . . . . . . . . . . . . . . . . . . . . . 80',\n",
       "  '10.3 Matrix power . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 81',\n",
       "  '10.4 QR factorization . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 82',\n",
       "  '',\n",
       "  '11 Matrix inverses 85',\n",
       "  '11.1 Left and right inverses . . . . . . . . . . . . . . . . . . . . . . . . . . . 85',\n",
       "  '11.2 Inverse . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 86',\n",
       "  '11.3 Solving linear equations . . . . . . . . . . . . . . . . . . . . . . . . . . 87',\n",
       "  '11.4 Examples . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 90',\n",
       "  '11.5 Pseudo-inverse . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 91',\n",
       "  '',\n",
       "  '12 Least squares 93',\n",
       "  '12.1 Least squares problem . . . . . . . . . . . . . . . . . . . . . . . . . . . 93',\n",
       "  '12.2 Solution . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 94',\n",
       "  '12.3 Solving least squares problems . . . . . . . . . . . . . . . . . . . . . . 95',\n",
       "  '12.4 Examples . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 96',\n",
       "  '',\n",
       "  '13 Least squares data fitting 99',\n",
       "  '13.1 Least squares data fitting . . . . . . . . . . . . . . . . . . . . . . . . . 99',\n",
       "  '13.2 Validation . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 105',\n",
       "  '13.3 Feature engineering . . . . . . . . . . . . . . . . . . . . . . . . . . . . 108',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Contents v',\n",
       "  '',\n",
       "  '14 Least squares classification 113',\n",
       "  '14.1 Classification . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 113',\n",
       "  '14.2 Least squares classifier . . . . . . . . . . . . . . . . . . . . . . . . . . . 114',\n",
       "  '14.3 Multi-class classifiers . . . . . . . . . . . . . . . . . . . . . . . . . . . 115',\n",
       "  '',\n",
       "  '15 Multi-objective least squares 121',\n",
       "  '15.1 Multi-objective least squares . . . . . . . . . . . . . . . . . . . . . . . 121',\n",
       "  '15.2 Control . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 124',\n",
       "  '15.3 Estimation and inversion . . . . . . . . . . . . . . . . . . . . . . . . . 124',\n",
       "  '15.4 Regularized data fitting . . . . . . . . . . . . . . . . . . . . . . . . . . 125',\n",
       "  '15.5 Complexity . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 127',\n",
       "  '',\n",
       "  '16 Constrained least squares 129',\n",
       "  '16.1 Constrained least squares problem . . . . . . . . . . . . . . . . . . . . 129',\n",
       "  '16.2 Solution . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 131',\n",
       "  '16.3 Solving contrained least squares problems . . . . . . . . . . . . . . . . 132',\n",
       "  '',\n",
       "  '17 Constrained least squares applications 135',\n",
       "  '17.1 Portfolio optimization . . . . . . . . . . . . . . . . . . . . . . . . . . . 135',\n",
       "  '17.2 Linear quadratic control . . . . . . . . . . . . . . . . . . . . . . . . . . 137',\n",
       "  '17.3 Linear quadratic state estimation . . . . . . . . . . . . . . . . . . . . . 143',\n",
       "  '',\n",
       "  '18 Nonlinear least squares 145',\n",
       "  '18.1 Nonlinear equations and least squares . . . . . . . . . . . . . . . . . . 145',\n",
       "  '18.2 Gauss–Newton algorithm . . . . . . . . . . . . . . . . . . . . . . . . . 145',\n",
       "  '18.3 Levenberg–Marquardt algorithm . . . . . . . . . . . . . . . . . . . . . 147',\n",
       "  '18.4 Nonlinear model fitting . . . . . . . . . . . . . . . . . . . . . . . . . . 152',\n",
       "  '18.5 Nonlinear least squares classification . . . . . . . . . . . . . . . . . . . 155',\n",
       "  '',\n",
       "  '19 Constrained nonlinear least squares 157',\n",
       "  '19.1 Constrained nonlinear least squares . . . . . . . . . . . . . . . . . . . . 157',\n",
       "  '19.2 Penalty algorithm . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 157',\n",
       "  '19.3 Augmented Lagrangian algorithm . . . . . . . . . . . . . . . . . . . . . 159',\n",
       "  '19.4 Nonlinear control . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 161',\n",
       "  '',\n",
       "  'Appendices 163',\n",
       "  '',\n",
       "  'A The VMLS package 165',\n",
       "  'A.1 Utility functions . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 165',\n",
       "  'A.2 Algorithms . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 166',\n",
       "  'A.3 Data sets . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 167',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'vi Contents',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Preface',\n",
       "  '',\n",
       "  'This Julia Language Companion accompanies our book Introduction to Applied',\n",
       "  'Linear Algebra: Vectors, Matrices, and Least Squares (referred to here as VMLS).',\n",
       "  'It is meant to show how the ideas and methods in VMLS can be expressed and',\n",
       "  'implemented in the programming language Julia.',\n",
       "  '',\n",
       "  'We assume that the reader has installed Julia, or is using Juliabox online, and',\n",
       "  'understands the basics of the language. We also assume that the reader has care-',\n",
       "  'fully read the relevant sections of VMLS. The organization of the Julia Language',\n",
       "  'Companion follows VMLS, section by section.',\n",
       "  '',\n",
       "  'You will see that mathematical notation and Julia syntax are pretty close, but',\n",
       "  'not the same. You must be careful to never confuse mathematical notation and',\n",
       "  'Julia syntax. In these notes we write mathematical notation (as in VMLS) in',\n",
       "  'standard mathematics font, e.g., y = Ax. Julia code, expressions, and snippets',\n",
       "  'are written in a fixed-width typewriter font, e.g., y = A*x. We encourage you to',\n",
       "  'cut and paste our Julia code snippets into a Julia interactive session or notebook,',\n",
       "  'to test them out, and maybe modify them and run them again. You can also use',\n",
       "  'our snippets as templates for your own Julia code. At some point we will collect',\n",
       "  'the Julia snippets in this companion document into Julia notebooks that you can',\n",
       "  'easily run.',\n",
       "  '',\n",
       "  'Julia is a very powerful language, but in this companion document we use only',\n",
       "  'a small and limited set of its features. The code snippets in this companion are',\n",
       "  'written so as to to be transparent and simple, and to emphasize closeness to the',\n",
       "  'concepts in VMLS. Some of the code snippets could be written in a much more',\n",
       "  'compact way, or in a way that would result in faster or more efficient execution.',\n",
       "  '',\n",
       "  'The code snippets in this compaion document are compatible with Julia 1.0,',\n",
       "  'which is not quite the same as earlier versions, like Julia 0.6. Some of the functions',\n",
       "  'we use are in the standard packages LinearAlgebra, SparseArrays, and Plots,',\n",
       "  'and a few others. We have created a simple and small Julia package VMLS, which',\n",
       "  'includes a small number of simple Julia functions that are useful for studying the',\n",
       "  'material in VMLS. The next section, Getting started with Julia, explains how to',\n",
       "  'install these Julia packages.',\n",
       "  '',\n",
       "  'We consider this companion to be a draft. We’ll be updating it occasionally,',\n",
       "  'adding more examples and fixing typos as we become aware of them. So you may',\n",
       "  'wish to periodically check whether a newer version is available.',\n",
       "  '',\n",
       "  'Stephen Boyd Stanford, California',\n",
       "  'Lieven Vandenberghe Los Angeles, California',\n",
       "  '',\n",
       "  ''],\n",
       " 'Empty',\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Getting started with Julia',\n",
       "  '',\n",
       "  'Installing Julia. Download Julia 1.0 from its website, and then follow the instruc-',\n",
       "  'tions to install it on your platform. You’ll want to make sure it’s working before',\n",
       "  'proceeding to install additional packages, as described below.',\n",
       "  '',\n",
       "  'Installing packages. While most of the Julia code we use in this companion is from',\n",
       "  'the base or core of the Julia language, several important functions are contained',\n",
       "  'in other packages that you must explicitly install. Here we explain how to do this.',\n",
       "  'The installation has to be done only once.',\n",
       "  '',\n",
       "  'To add (install) a Julia package you use the package manager system, which',\n",
       "  'is entered by typing ] to the Julia prompt. You’ll then see the package man-',\n",
       "  'ager prompt, (v.10) pkg>. If you type ?, you’ll see a list of package manager',\n",
       "  'commands. You exit the package manager by typing Ctrl-C (Control-C), which',\n",
       "  'returns you to the julia prompt.',\n",
       "  '',\n",
       "  'To add a package called PackageName, type add PackageName to the package',\n",
       "  'control prompt. It may take a few minutes to get the required packages, compile,',\n",
       "  'and install them. If you type status, you’ll see what packages are installed. If you',\n",
       "  'type up, installed packages will be upgraded, if needed.',\n",
       "  '',\n",
       "  'This companion will use functions from the following packages:',\n",
       "  '',\n",
       "  '• LinearAlgebra',\n",
       "  '',\n",
       "  '• SparseArrays',\n",
       "  '',\n",
       "  '• Plots',\n",
       "  '',\n",
       "  'On a few occasions we use functions from the packages Random and DSP, but we',\n",
       "  'will mention this in the text when we use these functions.',\n",
       "  '',\n",
       "  'Here’s what it looks like when you install the Plots package.',\n",
       "  '',\n",
       "  'julia> ]',\n",
       "  '',\n",
       "  '(v1.0) pkg> add Plots',\n",
       "  '',\n",
       "  'Updating registry at ‘~/.julia/registries/General‘',\n",
       "  '',\n",
       "  'Updating git-repo ‘https://github.com/JuliaRegistries/General.git‘',\n",
       "  '',\n",
       "  'Resolving package versions...',\n",
       "  '',\n",
       "  'Updating ‘~/.julia/environments/v1.0/Project.toml‘',\n",
       "  '',\n",
       "  '[no changes]',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'x Getting started with Julia',\n",
       "  '',\n",
       "  'Updating ‘~/.julia/environments/v1.0/Manifest.toml‘',\n",
       "  '',\n",
       "  '[no changes]',\n",
       "  '',\n",
       "  '(v1.0) pkg> status',\n",
       "  '',\n",
       "  'Status ‘~/.julia/environments/v1.0/Project.toml‘',\n",
       "  '',\n",
       "  '[91a5bcdd] Plots v0.19.3',\n",
       "  '',\n",
       "  '[011e45e9] SparseArrays v0.4.2',\n",
       "  '',\n",
       "  '[37e2e46d] LinearAlgebra',\n",
       "  '',\n",
       "  '(v1.0) pkg> ^C',\n",
       "  '',\n",
       "  'julia>',\n",
       "  '',\n",
       "  'The VMLS package. We have created a small package called VMLS. It contains a',\n",
       "  'few functions that use notation closer to VMLS notation or are easier to use than',\n",
       "  'the corresponding Julia functions, basic implementations of some algorithms in',\n",
       "  'VMLS, and functions that generate data used in examples. The list of functions is',\n",
       "  'given in Appendix A. To install VMLS, go to the package manager from the Julia',\n",
       "  'prompt, then install it as follows.',\n",
       "  '',\n",
       "  'julia> ]',\n",
       "  '',\n",
       "  'pkg> add https://github.com/VMLS-book/VMLS.jl',\n",
       "  '',\n",
       "  'pkg> ^C',\n",
       "  '',\n",
       "  'julia>',\n",
       "  '',\n",
       "  'Using packages. Once a package is installed (which needs to be done only once),',\n",
       "  'you import it into Julia with the command using, followed by the package name,',\n",
       "  'or a comma separated list of package names. This too can take some time. After',\n",
       "  'executing this command you can access the functions contained in the packages.',\n",
       "  '',\n",
       "  'To run any the code fragments in this companion you will need to first execute',\n",
       "  'the statement',\n",
       "  '',\n",
       "  'julia> using LinearAlgebra, SparseArrays, VMLS',\n",
       "  '',\n",
       "  'When we use other packages (in particular, Plots), we include the using statement',\n",
       "  'in the code.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Chapter 1',\n",
       "  '',\n",
       "  'Vectors',\n",
       "  '',\n",
       "  '1.1 Vectors',\n",
       "  '',\n",
       "  'Vectors in Julia are represented by one-dimensional Array objects. A vector is',\n",
       "  'constructed by giving the list of elements surrounded by square brackets, with the',\n",
       "  'elements separated by commas or semicolons. The assignment operator = is used',\n",
       "  'to give a name to the array. The length function returns the size (dimension).',\n",
       "  '',\n",
       "  'julia> x = [ -1.1, 0.0, 3.6, -7.2 ]',\n",
       "  '',\n",
       "  '4-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '-1.1',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '3.6',\n",
       "  '',\n",
       "  '-7.2',\n",
       "  '',\n",
       "  'julia> length(x)',\n",
       "  '',\n",
       "  '4',\n",
       "  '',\n",
       "  'julia> y = [ -1.1; 0.0; 3.6; -7.2 ] # Using semicolons',\n",
       "  '',\n",
       "  '4-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '-1.1',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '3.6',\n",
       "  '',\n",
       "  '-7.2',\n",
       "  '',\n",
       "  'julia> length(y)',\n",
       "  '',\n",
       "  '4',\n",
       "  '',\n",
       "  'The Array{Float64,1} displayed by Julia above each array tells us that the array',\n",
       "  'is one-dimensional and its entries are floating point numbers that use 64 bits.',\n",
       "  '',\n",
       "  'Some common mistakes. Don’t forget the commas or semicolons between entries,',\n",
       "  'and be sure to use square brackets and not parentheses. Otherwise you’ll get things',\n",
       "  'that makes sense in Julia, but are not vectors.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '2 1 Vectors',\n",
       "  '',\n",
       "  'julia> a = [ 1 2 ]',\n",
       "  '',\n",
       "  '1×2 Array{Int64,2}:',\n",
       "  '1 2',\n",
       "  '',\n",
       "  'julia> b = ( 1, 2 )',\n",
       "  '',\n",
       "  '(1, 2)',\n",
       "  '',\n",
       "  'Here a is a row vector, which we will encounter later; b is a tuple or list consisting',\n",
       "  'of two scalars.',\n",
       "  '',\n",
       "  'Indexing. A specific element xi is retrieved by the expression x[i] where i is the',\n",
       "  'index (which runs to 1 to n, for an n-vector). Array indexing can be also be used',\n",
       "  'on the left-hand side of an assignment, to change the value of a specific element.',\n",
       "  '',\n",
       "  'julia> x = [ -1.1, 0.0, 3.6, -7.2 ];',\n",
       "  '',\n",
       "  'julia> x[3]',\n",
       "  '',\n",
       "  '3.6',\n",
       "  '',\n",
       "  'julia> x[3] = 4.0;',\n",
       "  '',\n",
       "  'julia> x',\n",
       "  '',\n",
       "  '4-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '-1.1',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '4.0',\n",
       "  '',\n",
       "  '-7.2',\n",
       "  '',\n",
       "  'The special index end refers to the last index of a vector. In the example above,',\n",
       "  'x[end] and x[length(x)] both give the last entry, -7.2.',\n",
       "  '',\n",
       "  'Assignment versus copying. Matlab or Octave users may be surprised by the',\n",
       "  'behavior of an assignment y = x if x is an array. This expression gives a new name',\n",
       "  '(or reference) y to the same array already referenced by x. It does not create a',\n",
       "  'new copy of the array x.',\n",
       "  '',\n",
       "  'julia> x = [ -1.1, 0.0, 3.6, -7.2 ];',\n",
       "  '',\n",
       "  'julia> y = x;',\n",
       "  '',\n",
       "  'julia> x[3] = 4.0;',\n",
       "  '',\n",
       "  'julia> y',\n",
       "  '',\n",
       "  '4-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '-1.1',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '4.0 # The assignment to x[3] also changed y[3]',\n",
       "  '',\n",
       "  '-7.2',\n",
       "  '',\n",
       "  'julia> y[1] = 2.0;',\n",
       "  '',\n",
       "  'julia> x',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '1.1 Vectors 3',\n",
       "  '',\n",
       "  '4-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '2.0 # The assignment to y[1] also changed x[1]',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '4.0',\n",
       "  '',\n",
       "  '-7.2',\n",
       "  '',\n",
       "  'To create a new copy of an array, the function copy should be used.',\n",
       "  '',\n",
       "  'julia> x = [ -1.1, 0.0, 3.6, -7.2 ];',\n",
       "  '',\n",
       "  'julia> y = copy(x);',\n",
       "  '',\n",
       "  'julia> x[3] = 4.0;',\n",
       "  '',\n",
       "  'julia> y',\n",
       "  '',\n",
       "  '4-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '-1.1',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '3.6',\n",
       "  '',\n",
       "  '-7.2',\n",
       "  '',\n",
       "  'julia> y[1] = 2.0;',\n",
       "  '',\n",
       "  'julia> x',\n",
       "  '',\n",
       "  '4-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '-1.1',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '4.0',\n",
       "  '',\n",
       "  '-7.2',\n",
       "  '',\n",
       "  'Vector equality. Equality of vectors is checked using the relational operator ==.',\n",
       "  'For two vectors (arrays) a and b, the Julia expression a==b evaluates to true if',\n",
       "  'the vectors (arrays) are equal, i.e., they have the same length and identical entries,',\n",
       "  'and false otherwise.',\n",
       "  '',\n",
       "  'julia> x = [ -1.1, 0.0, 3.6, -7.2 ];',\n",
       "  '',\n",
       "  'julia> y = copy(x);',\n",
       "  '',\n",
       "  'julia> y[3] = 4.0',\n",
       "  '',\n",
       "  'julia> y == x',\n",
       "  '',\n",
       "  'false',\n",
       "  '',\n",
       "  'julia> z = x',\n",
       "  '',\n",
       "  'julia> z[3] = 4.0',\n",
       "  '',\n",
       "  'julia> z == x',\n",
       "  '',\n",
       "  'true',\n",
       "  '',\n",
       "  'Scalars versus 1-vectors. In the mathematical notation used in VMLS we con-',\n",
       "  'sider a 1-vector to be the same as a number. But in Julia, 1-vectors are not the same',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '4 1 Vectors',\n",
       "  '',\n",
       "  'as scalars (numbers). Julia distinguishes between the 1-vector (array) [ 1.3 ] and',\n",
       "  'the number 1.3.',\n",
       "  '',\n",
       "  'julia> x = [ 1.3 ]',\n",
       "  '',\n",
       "  '1-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '1.3',\n",
       "  '',\n",
       "  'julia> y = 1.3',\n",
       "  '',\n",
       "  '1.3',\n",
       "  '',\n",
       "  'julia> x == y',\n",
       "  '',\n",
       "  'false',\n",
       "  '',\n",
       "  'julia> x[1] == y',\n",
       "  '',\n",
       "  'true',\n",
       "  '',\n",
       "  'In the last line, x[1] is the first (and only) entry of x, which is indeed the number',\n",
       "  '1.3.',\n",
       "  '',\n",
       "  'Block or stacked vectors. To construct a block vector in Julia, you can use vcat',\n",
       "  '(vertical concatenate) or the semicolon (;) operator. Let’s construct the block',\n",
       "  'vector z = (x, y) with x = (1,−2) and y = (1, 1, 0) using the two methods.',\n",
       "  '',\n",
       "  'julia> x = [ 1, -2 ]; y = [ 1, 1, 0 ];',\n",
       "  '',\n",
       "  'julia> z = [ x; y ] # Concatenate using semicolon',\n",
       "  '',\n",
       "  '5-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '-2',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  'julia> z = vcat(x, y) # Concatenate using vcat',\n",
       "  '',\n",
       "  '5-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '-2',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  'As in mathematical notation, you can stack vectors with scalars, e.g., [1;x;0]',\n",
       "  'creates (1, x, 0).',\n",
       "  '',\n",
       "  'Some common mistakes. There are a few Julia operations that look similar but',\n",
       "  'do not construct a block or stacked vector. For example, z = (x,y) creates a list',\n",
       "  'or tuple of the two vectors; z = [x,y] creates an array of the two vectors. Both of',\n",
       "  'these are valid Julia expression, but neither of them is the stacked vector [x;y].',\n",
       "  '',\n",
       "  'Subvectors and slicing. As in the mathematical notation used in VMLS, the Julia',\n",
       "  'expression r:s denotes the index range r, r + 1, . . . , s. (It is assumed here that r',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '1.1 Vectors 5',\n",
       "  '',\n",
       "  'and s are positive integers with r the smaller of the two.) In VMLS, we use xr:s',\n",
       "  'to denote the slice of the vector x from index r to s. In Julia you can extract a',\n",
       "  'subvector or slice of a vector using an index range as the argument. You can also',\n",
       "  'use index ranges to assign a slice of a vector.',\n",
       "  '',\n",
       "  'julia> x = [ 9, 4, 3, 0, 5 ];',\n",
       "  '',\n",
       "  'julia> y = x[2:4]',\n",
       "  '',\n",
       "  '3-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '4',\n",
       "  '',\n",
       "  '3',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  'julia> x[4:5] = [ -2, -3 ]; # Re-assign the 4 and 5 entries of x',\n",
       "  '',\n",
       "  'julia> x',\n",
       "  '',\n",
       "  '5-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '9',\n",
       "  '',\n",
       "  '4',\n",
       "  '',\n",
       "  '3',\n",
       "  '',\n",
       "  '-2',\n",
       "  '',\n",
       "  '-3',\n",
       "  '',\n",
       "  'Julia indexing into arrays. Julia slicing and subvectoring is much more general',\n",
       "  'than the mathematical notation we use in VMLS. For example, one can use a',\n",
       "  'number range with a third argument, that gives the stride, which is the increment',\n",
       "  'between successive indexes. For example, the index range 1:2:5 is the list of',\n",
       "  'numbers 1,3,5. The expression x[1:2:5] extracts the 3-vector [9,3,5], i.e., the',\n",
       "  'first, third, and fifth entries of x defined above. You can also use an index range',\n",
       "  'that runs backward. For any vector z, the Julia expression z[end:-1:1] is the',\n",
       "  'reversed vector, i.e., the vector with the same coefficients, but in opposite order.',\n",
       "  '',\n",
       "  'Vector of first differences. Let’s use slicing to create the (n−1)-vector d defined',\n",
       "  'by di = xi+1 − xi, for i = 1, . . . , n − 1, where x is an n-vector. The vector d is',\n",
       "  'called the vector of (first) differences of x.',\n",
       "  '',\n",
       "  'julia> x = [ 1, 0, 0, -2, 2 ];',\n",
       "  '',\n",
       "  'julia> d = x[2:end] - x[1:end-1]',\n",
       "  '',\n",
       "  '4-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '-1',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  '-2',\n",
       "  '',\n",
       "  '4',\n",
       "  '',\n",
       "  'Lists of vectors. An ordered list of n-vectors might be denoted in VMLS as',\n",
       "  'a1, . . . , ak or a',\n",
       "  '',\n",
       "  '(1), . . . , a(k), or just as a, b, c. There are several ways to represent',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '6 1 Vectors',\n",
       "  '',\n",
       "  'lists of vectors in Julia. If we give the elements of the list, separated by commas,',\n",
       "  'and surrounded by square brackets, we form a one-dimensional array of vectors. If',\n",
       "  'instead we use parentheses as delimiters, we obtain a tuple or list.',\n",
       "  '',\n",
       "  'julia> x = [ 1.0, 0 ]; y = [ 1.0, -1.0 ]; z = [ 0, 1.0];',\n",
       "  '',\n",
       "  'julia> list = [ x, y, z ]',\n",
       "  '',\n",
       "  '3-element Array{Array{Float64,1},1}:',\n",
       "  '',\n",
       "  '[1.0, 0.0]',\n",
       "  '',\n",
       "  '[1.0, -1.0]',\n",
       "  '',\n",
       "  '[0.0, 1.0]',\n",
       "  '',\n",
       "  'julia> list[2] # Second element of list',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '1.0',\n",
       "  '',\n",
       "  '-1.0',\n",
       "  '',\n",
       "  'julia> list = ( x, y, z )',\n",
       "  '',\n",
       "  '([1.0, 0.0], [1.0, -1.0], [0.0, 1.0])',\n",
       "  '',\n",
       "  'julia> list[3] # Third element of list',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '1.0',\n",
       "  '',\n",
       "  'Note the difference between [x, y, z] (an array of arrays) and [x; y; z] (an',\n",
       "  'array of numbers, obtained by concatenation). To extract the ith vector from the',\n",
       "  'list of vectors, use list[i]. To get the jth element or coefficient of the ith vector',\n",
       "  'in the list, use list[i][j].',\n",
       "  '',\n",
       "  'Zero vectors. In Julia a zero vector of dimension n is created using zeros(n).',\n",
       "  '',\n",
       "  'julia> zeros(3)',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  'The expression zeros(length(a)) creates a zero vector with the same size as the',\n",
       "  'vector a.',\n",
       "  '',\n",
       "  'Unit vectors. There is no built-in Julia function for creating ei, the ith unit vector',\n",
       "  'of length n. The following code creates ei, with i = 2 and n = 4.',\n",
       "  '',\n",
       "  'julia> i = 2; n = 4;',\n",
       "  '',\n",
       "  'julia> ei = zeros(n); # Create a zero vector',\n",
       "  '',\n",
       "  'julia> ei[i] = 1; # Set ith entry to 1',\n",
       "  '',\n",
       "  'julia> ei',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '1.1 Vectors 7',\n",
       "  '',\n",
       "  '4-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '1.0',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  'Here’s another way to create ei using concatenation, using a Julia inline func-',\n",
       "  'tion.',\n",
       "  '',\n",
       "  'julia> unit_vector(i,n) = [zeros(i-1); 1 ; zeros(n-i)]',\n",
       "  '',\n",
       "  'unit_vector (generic function with 1 method)',\n",
       "  '',\n",
       "  'julia> unit_vector(2,4)',\n",
       "  '',\n",
       "  '4-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '1.0',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  'Ones vector. In Julia the ones vector of dimension n, denoted 1n or just 1 in',\n",
       "  'VMLS, is created using ones(n).',\n",
       "  '',\n",
       "  'julia> ones(2)',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '1.0',\n",
       "  '',\n",
       "  '1.0',\n",
       "  '',\n",
       "  'Random vectors. We do not use or refer to random vectors in VMLS, which',\n",
       "  'does not assume a background in probability. However, it is sometimes useful',\n",
       "  'to generate random vectors, for example to test an identity or some algorithm. In',\n",
       "  'Julia, rand(n) generates a random vector of length n with entries that are between',\n",
       "  '0 and 1. Each time this function is called or evaluated, it gives a different vector.',\n",
       "  'The variant randn(n) (with the extra ‘n’ for normal) gives an n-vector with entries',\n",
       "  'that come from a normal (Gaussian) distribution. They can be positive or negative,',\n",
       "  'with typical values on the order of one. Remember that every time you evaluate',\n",
       "  'these functions, you get a different random vector. In particular, you will obtain',\n",
       "  'different entries in the vectors below when you run the code.',\n",
       "  '',\n",
       "  'julia> rand(2)',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.831491',\n",
       "  '',\n",
       "  '0.0497708',\n",
       "  '',\n",
       "  'julia> rand(2)',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '8 1 Vectors',\n",
       "  '',\n",
       "  '0 10 20 30 40',\n",
       "  '',\n",
       "  '70',\n",
       "  '',\n",
       "  '75',\n",
       "  '',\n",
       "  '80',\n",
       "  '',\n",
       "  '85',\n",
       "  '',\n",
       "  'Figure 1.1 Hourly temperature in downtown Los Angeles on August 5 and',\n",
       "  '6, 2015 (starting at 12:47AM, ending at 11:47PM).',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.189284',\n",
       "  '',\n",
       "  '0.713467',\n",
       "  '',\n",
       "  'julia> randn(2)',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '2.44544',\n",
       "  '',\n",
       "  '-0.12134',\n",
       "  '',\n",
       "  'Plotting. There are several external packages for creating plots in Julia. One such',\n",
       "  'package is Plots.jl, which you must add (install) via Julia’s package manager',\n",
       "  'control system; see page ix. Assuming the Plots package had been installed, you',\n",
       "  'import it into Julia for use, using the command using Plots. (This can take some',\n",
       "  'time.) After that you can access the Julia commands that create or manipulate',\n",
       "  'plots.',\n",
       "  '',\n",
       "  'For example, we can plot the temperature time series in Figure 1.3 of VMLS',\n",
       "  'using the code below; the last line saves the plot in the file temperature.pdf. The',\n",
       "  'result is shown in Figure 1.1.',\n",
       "  '',\n",
       "  'julia> using Plots # Only need to do this once per session',\n",
       "  '',\n",
       "  '[ Info: Precompiling Plots [91a5bcdd-55d7-5caf-9e0b-520d859cae80]',\n",
       "  '',\n",
       "  '[ Info: Precompiling GR [28b8d3ca-fb5f-59d9-8090-bfdbd6d07a71]',\n",
       "  '',\n",
       "  'julia> temps = [ 71, 71, 68, 69, 68, 69, 68, 74, 77, 82, 85, 86,',\n",
       "  '',\n",
       "  '88, 86, 85, 86, 84, 79, 77, 75, 73, 71, 70, 70, 69, 69, 69,',\n",
       "  '',\n",
       "  '69, 67, 68, 68, 73, 76, 77, 82, 84, 84, 81, 80, 78, 79, 78,',\n",
       "  '',\n",
       "  '73, 72, 70, 70, 68, 67 ];',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.1.3',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '1.2 Vector addition 9',\n",
       "  '',\n",
       "  'julia> plot(temps, marker = :circle, legend = false, grid = false)',\n",
       "  '',\n",
       "  'julia> savefig(\"temperature.pdf\")',\n",
       "  '',\n",
       "  '1.2 Vector addition',\n",
       "  '',\n",
       "  'Vector addition and subtraction. If x and y are vectors of the same size, x+y and',\n",
       "  'x-y give their sum and difference, respectively.',\n",
       "  '',\n",
       "  'julia> [ 0, 7, 3 ] + [ 1, 2, 0 ] # Vector addition',\n",
       "  '',\n",
       "  '3-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '9',\n",
       "  '',\n",
       "  '3',\n",
       "  '',\n",
       "  'julia> [ 1, 9 ] - [ 1, 1] # Vector subtraction',\n",
       "  '',\n",
       "  '2-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  '8',\n",
       "  '',\n",
       "  '1.3 Scalar-vector multiplication',\n",
       "  '',\n",
       "  'Scalar-vector multiplication and division. If a is a number and x a vector, you',\n",
       "  'can express the scalar-vector product either as a*x or x*a. (Julia actually allows',\n",
       "  'you to write 2.0x for 2.0*x. This is unambiguous because variable names cannot',\n",
       "  'start with a number.) You can carry out scalar-vector division in a similar way, as',\n",
       "  'x/a, or the less familar looking expression a\\\\x.',\n",
       "  '',\n",
       "  'julia> x = [ 0, 2, -1 ];',\n",
       "  '',\n",
       "  'julia> 2.2 * x # Scalar-vector multiplication',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '4.4',\n",
       "  '',\n",
       "  '-2.2',\n",
       "  '',\n",
       "  'julia> x * 2.2 # Scalar-vector multiplication',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '4.4',\n",
       "  '',\n",
       "  '-2.2',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '10 1 Vectors',\n",
       "  '',\n",
       "  'julia> x / 3 # Scalar-vector division',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '0.666667',\n",
       "  '',\n",
       "  '-0.333333',\n",
       "  '',\n",
       "  'julia> 3 \\\\ x # Scalar-vector division',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '0.666667',\n",
       "  '',\n",
       "  '-0.333333',\n",
       "  '',\n",
       "  'Scalar-vector addition. In Julia you can add a scalar a and a vector x using',\n",
       "  'x .+ a. The dot that precedes the plus symbol tells Julia to apply the operation',\n",
       "  'to each element. (More on this below.) The meaning is that the scalar is added',\n",
       "  'or subtracted to each element. (This is not standard mathematical notation; in',\n",
       "  'VMLS we denote this as, e.g., x+ a1, where x is an n-vector and a is a scalar.) In',\n",
       "  'Julia you can also carry out scalar-vector addition with the scalar on the left.',\n",
       "  '',\n",
       "  'julia> [ 1.1, -3.7, 0.3 ] .- 1.4 # Vector-scalar subtraction',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '-0.3',\n",
       "  '',\n",
       "  '-5.1',\n",
       "  '',\n",
       "  '-1.1',\n",
       "  '',\n",
       "  'julia> 0.7 .+ [1,-1]',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '1.7',\n",
       "  '',\n",
       "  '-0.30000000000000004',\n",
       "  '',\n",
       "  'Elementwise operations. Julia supports methods for carrying out an operation',\n",
       "  'on every element or coefficient of a vector. To do this we add a period or dot',\n",
       "  'before the operator. For example, if x and y are vectors of the same length, then',\n",
       "  'x.*y, x./y, x.\\\\y, x.^y are elementwise vector-vector operations. They result in',\n",
       "  'vectors of the same length as x and y, and ith element xiyi, xi/yi, yi/xi, and x',\n",
       "  '',\n",
       "  'yi',\n",
       "  'i ,',\n",
       "  '',\n",
       "  'respectively.',\n",
       "  'As an example of elementwise division, let’s find the 3-vector of asset returns r',\n",
       "  '',\n",
       "  'from the (vectors of) initial and final prices of assets (see page 22 in VMLS).',\n",
       "  '',\n",
       "  'julia> p_initial = [ 22.15, 89.32, 56.77 ];',\n",
       "  '',\n",
       "  'julia> p_final = [ 23.05, 87.32, 57.13 ];',\n",
       "  '',\n",
       "  'julia> r = (p_final - p_initial) ./ p_initial',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.0406321',\n",
       "  '',\n",
       "  '-0.0223914',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.1.13',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '1.3 Scalar-vector multiplication 11',\n",
       "  '',\n",
       "  '0.00634138',\n",
       "  '',\n",
       "  'Elementwise operations with a scalar. Elementwise operations work when one',\n",
       "  'of the arguments is a scalar, in which case it is interpreted as the scalar times a',\n",
       "  'ones vector of the appropriate dimension. Scalar-vector addition, described above,',\n",
       "  'is a special case of this. If a is a scalar and x is a vector, then x.^a is a vector with',\n",
       "  'ith element xai , and a.^x is a vector with elements a',\n",
       "  '',\n",
       "  'xi . Like scalar-vector addition,',\n",
       "  'the dot notation for elementwise operations is not standard mathematical notation',\n",
       "  'and we do not use it in VMLS.',\n",
       "  '',\n",
       "  'We can also use the period notation with a function that has a name, to let',\n",
       "  'Julia know that the function should be applied elementwise. In this case we add the',\n",
       "  'period after the function name to indicate that it should be applied elementwise.',\n",
       "  'For example, if x is a vector, we can form sin.(x) to apply the sine function to',\n",
       "  'each element of x.',\n",
       "  '',\n",
       "  'The equality test operator == (and other relational operators like <, >=) can be',\n",
       "  'made to work elementwise by preceding it with a period: x==y tells us whether or',\n",
       "  'not x and y are the same vector; x.==y is a vector whose entries tell us whether',\n",
       "  'the corresponding entries in x and y are the same.',\n",
       "  '',\n",
       "  'julia> w = [1,2,2]; z = [1,2,3];',\n",
       "  '',\n",
       "  'julia> w == z',\n",
       "  '',\n",
       "  'false',\n",
       "  '',\n",
       "  'julia> w .== z',\n",
       "  '',\n",
       "  '3-element BitArray{1}:',\n",
       "  '',\n",
       "  'true',\n",
       "  '',\n",
       "  'true',\n",
       "  '',\n",
       "  'false',\n",
       "  '',\n",
       "  'You can combine this with Julia’s slicing to extract the subvector of entries that',\n",
       "  'satisfy some logical condition. For example x[abs.(x).>1] gives the subvector of',\n",
       "  'x consisting of the entries larger than one in magnitude.',\n",
       "  '',\n",
       "  'julia> x = [1.1, .5, -1.5, -0.3]',\n",
       "  '',\n",
       "  '4-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '1.1',\n",
       "  '',\n",
       "  '0.5',\n",
       "  '',\n",
       "  '-1.5',\n",
       "  '',\n",
       "  '-0.3',\n",
       "  '',\n",
       "  'julia> x[abs.(x) .> 1]',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '1.1',\n",
       "  '',\n",
       "  '-1.5',\n",
       "  '',\n",
       "  'Dot notation works with assignment too, allowing you to assign multiple entries',\n",
       "  'of a vector to a scalar value. For example:',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '12 1 Vectors',\n",
       "  '',\n",
       "  'julia> x = rand(4)',\n",
       "  '',\n",
       "  '4-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.4735758513909343',\n",
       "  '',\n",
       "  '0.3554729725184458',\n",
       "  '',\n",
       "  '0.13775623085957855',\n",
       "  '',\n",
       "  '0.9227200780245117',\n",
       "  '',\n",
       "  'julia> x[1:2] = [-1,1];',\n",
       "  '',\n",
       "  'julia> x',\n",
       "  '',\n",
       "  '4-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '-1.0',\n",
       "  '',\n",
       "  '1.0',\n",
       "  '',\n",
       "  '0.13775623085957855',\n",
       "  '',\n",
       "  '0.9227200780245117',\n",
       "  '',\n",
       "  'julia> x[2:3] .= 1.3;',\n",
       "  '',\n",
       "  'julia> x',\n",
       "  '',\n",
       "  '4-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '-1.0',\n",
       "  '',\n",
       "  '1.3',\n",
       "  '',\n",
       "  '1.3',\n",
       "  '',\n",
       "  '3.0',\n",
       "  '',\n",
       "  'Linear combination. You can form a linear combination is Julia using scalar-',\n",
       "  'vector multiplication and addition.',\n",
       "  '',\n",
       "  'julia> a = [ 1, 2 ]; b = [ 3, 4 ];',\n",
       "  '',\n",
       "  'julia> alpha = -0.5; beta = 1.5;',\n",
       "  '',\n",
       "  'julia> c = alpha*a + beta*b',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '4.0',\n",
       "  '',\n",
       "  '5.0',\n",
       "  '',\n",
       "  'To illustrate some additional Julia syntax, we create a function that takes a',\n",
       "  'list of coefficients and a list of vectors as its arguments, and returns the linear',\n",
       "  'combination. The lists can be represented by tuples or arrays.',\n",
       "  '',\n",
       "  'julia> function lincomb(coeff, vectors)',\n",
       "  '',\n",
       "  'n = length(vectors[1]) # Length of vectors',\n",
       "  '',\n",
       "  'a = zeros(n);',\n",
       "  '',\n",
       "  'for i = 1:length(vectors)',\n",
       "  '',\n",
       "  'a = a + coeff[i] * vectors[i];',\n",
       "  '',\n",
       "  'end',\n",
       "  '',\n",
       "  'return a',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '1.3 Scalar-vector multiplication 13',\n",
       "  '',\n",
       "  'end',\n",
       "  '',\n",
       "  'lincomb (generic function with 1 method)',\n",
       "  '',\n",
       "  'julia> lincomb( ( -0.5, 1.5), ( [1, 2], [ 3, 4]) )',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '4.0',\n",
       "  '',\n",
       "  '5.0',\n",
       "  '',\n",
       "  'A more concise definition of the function is as follows.',\n",
       "  '',\n",
       "  'julia> function lincomb(coeff, vectors)',\n",
       "  '',\n",
       "  'return sum( coeff[i] * vectors[i] for i = 1:length(vectors) )',\n",
       "  '',\n",
       "  'end',\n",
       "  '',\n",
       "  'Checking properties. Let’s check the distributive property',\n",
       "  '',\n",
       "  'β(a+ b) = βa+ βb,',\n",
       "  '',\n",
       "  'which holds for any two n-vectors a and b, and any scalar β. We’ll do this for',\n",
       "  'n = 3, and randomly generated a, b, and β. (This computation does not show',\n",
       "  'that the property always holds; it only shows that it holds for the specific vectors',\n",
       "  'chosen. But it’s good to be skeptical and check identities with random arguments.)',\n",
       "  'We use the lincomb function we just defined.',\n",
       "  '',\n",
       "  'julia> a = rand(3)',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.55304',\n",
       "  '',\n",
       "  '0.55801',\n",
       "  '',\n",
       "  '0.0299682',\n",
       "  '',\n",
       "  'julia> b = rand(3);',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.796619',\n",
       "  '',\n",
       "  '0.578865',\n",
       "  '',\n",
       "  '0.219901',\n",
       "  '',\n",
       "  'julia> beta = randn() # Generates a random scalar',\n",
       "  '',\n",
       "  'beta = randn()',\n",
       "  '',\n",
       "  '-0.17081925677011056',\n",
       "  '',\n",
       "  'julia> lhs = beta*(a+b)',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '-0.230548',\n",
       "  '',\n",
       "  '-0.1942',\n",
       "  '',\n",
       "  '-0.0426825',\n",
       "  '',\n",
       "  'julia> rhs = beta*a + beta*b',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '14 1 Vectors',\n",
       "  '',\n",
       "  '-0.230548',\n",
       "  '',\n",
       "  '-0.1942',\n",
       "  '',\n",
       "  '-0.0426825',\n",
       "  '',\n",
       "  'Although the two vectors lhs and rhs are displayed as the same, they might not be',\n",
       "  'exactly the same, due to very small round-off errors in floating point computations.',\n",
       "  'When we check an identity using random numbers, we can expect that the left-',\n",
       "  'hand and right-hand sides of the identity are not exactly the same, but very close',\n",
       "  'to each other.',\n",
       "  '',\n",
       "  '1.4 Inner product',\n",
       "  '',\n",
       "  'Inner product. The inner product of n-vectors x and y is denoted as xT y. In',\n",
       "  \"Julia, the inner product of x and y is denoted as x'*y.\",\n",
       "  '',\n",
       "  'julia> x = [ -1, 2, 2 ];',\n",
       "  '',\n",
       "  'julia> y = [ 1, 0, -3 ];',\n",
       "  '',\n",
       "  \"julia> x'*y\",\n",
       "  '-7',\n",
       "  '',\n",
       "  'Net present value. As an example, the following code snippet finds the net',\n",
       "  'present value (NPV) of a cash flow vector c, with per-period interest rate r.',\n",
       "  '',\n",
       "  'julia> c = [ 0.1, 0.1, 0.1, 1.1 ]; # Cash flow vector',\n",
       "  '',\n",
       "  'julia> n = length(c);',\n",
       "  '',\n",
       "  'julia> r = 0.05; # 5% per-period interest rate',\n",
       "  '',\n",
       "  'julia> d = (1+r) .^ -(0:n-1)',\n",
       "  '',\n",
       "  '4-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '1.0',\n",
       "  '',\n",
       "  '0.952381',\n",
       "  '',\n",
       "  '0.907029',\n",
       "  '',\n",
       "  '0.863838',\n",
       "  '',\n",
       "  \"julia> NPV = c'*d\",\n",
       "  '1.236162401468524',\n",
       "  '',\n",
       "  'In the fourth line, to get the vector d we raise the scalar 1+r elementwise to the',\n",
       "  'powers given in the array (0:n-1), which expands to [0,1,...,n-1].',\n",
       "  '',\n",
       "  'Total school-age population. Suppose that the 100-vector x gives the age dis-',\n",
       "  'tribution of some population, with xi the number of people of age i − 1, for',\n",
       "  'i = 1, . . . , 100. The total number of people with age between 5 and 18 (inclu-',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '1.5 Complexity of vector computations 15',\n",
       "  '',\n",
       "  'sive) is given by',\n",
       "  '',\n",
       "  'x6 + x7 + · · ·+ x18 + x19.',\n",
       "  '',\n",
       "  'We can express this as sTx, where s is the vector with entries one for i = 6, . . . , 19',\n",
       "  'and zero otherwise. In Julia, this is expressed as',\n",
       "  '',\n",
       "  'julia> s = [ zeros(5); ones(14); zeros(81) ];',\n",
       "  '',\n",
       "  \"julia> school_age_pop = s'*x\",\n",
       "  '',\n",
       "  'Several other expressions can be used to evaluate this quantity, for example, the',\n",
       "  'expression sum(x[6:19]), using the Julia function sum, which gives the sum of the',\n",
       "  'entries of a vector.',\n",
       "  '',\n",
       "  '1.5 Complexity of vector computations',\n",
       "  '',\n",
       "  'Floating point operations. For any two numbers a and b, we have (a+b)(a−b) =',\n",
       "  'a2− b2. When a computer calculates the left-hand and right-hand side, for specific',\n",
       "  'numbers a and b, they need not be exactly the same, due to very small floating',\n",
       "  'point round-off errors. But they should be very nearly the same. Let’s see an',\n",
       "  'example of this.',\n",
       "  '',\n",
       "  'julia> a = rand(); b = rand();',\n",
       "  '',\n",
       "  'julia> lhs = (a+b) * (a-b)',\n",
       "  '',\n",
       "  '-0.025420920298883976',\n",
       "  '',\n",
       "  'julia> rhs = a^2 - b^2',\n",
       "  '',\n",
       "  '-0.02542092029888398',\n",
       "  '',\n",
       "  'julia> lhs - rhs',\n",
       "  '',\n",
       "  '3.469446951953614e-18',\n",
       "  '',\n",
       "  'Here we see that the left-hand and right-hand sides are not exactly equal, but very',\n",
       "  'very close.',\n",
       "  '',\n",
       "  'Complexity. You can time a Julia command by adding @time before the com-',\n",
       "  'mand. The timer is not very accurate for very small times, say, measured in',\n",
       "  'microseconds (10−6 seconds). Also, you should run the command more than once;',\n",
       "  'it can be a lot faster on the second or subsequent runs.',\n",
       "  '',\n",
       "  'julia> a = randn(10^5); b = randn(10^5);',\n",
       "  '',\n",
       "  \"julia> @time a'*b\",\n",
       "  '0.002695 seconds (5 allocations: 176 bytes)',\n",
       "  '',\n",
       "  '38.97813069037062',\n",
       "  '',\n",
       "  \"julia> @time a'*b\",\n",
       "  '0.000173 seconds (5 allocations: 176 bytes)',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '16 1 Vectors',\n",
       "  '',\n",
       "  '38.97813069037062',\n",
       "  '',\n",
       "  'julia> c = randn(10^6); d = randn(10^6);',\n",
       "  '',\n",
       "  \"julia> @time c'*d\",\n",
       "  '0.001559 seconds (5 allocations: 176 bytes)',\n",
       "  '',\n",
       "  '1189.2960722446112',\n",
       "  '',\n",
       "  \"julia> @time c'*d\",\n",
       "  '0.001765 seconds (5 allocations: 176 bytes)',\n",
       "  '',\n",
       "  '1189.2960722446112',\n",
       "  '',\n",
       "  'The first inner product, of vectors of length 105, takes around 0.00017 seconds;',\n",
       "  'the second, with vectors of length 106 (tens times bigger), product takes around',\n",
       "  '0.0018 seconds, about 10 longer. This is predicted by the complexity of the inner',\n",
       "  'product, which is 2n − 1 flops. The computer on which the computations were',\n",
       "  'done is capable of around 2 · 106/0.001765 flops per second, i.e., around 1 Gflop/s.',\n",
       "  'These timings, and the estimate of computer speed, are very approximate.',\n",
       "  '',\n",
       "  'Sparse vectors. Functions for creating and manipulating sparse vectors are con-',\n",
       "  'tained in the Julia package SparseArrays, so you need to install this package before',\n",
       "  'you can use them; see page ix.',\n",
       "  '',\n",
       "  'Sparse vectors are stored as sparse arrays, i.e., arrays in which only the nonzero',\n",
       "  'elements are stored. In Julia you can create a sparse vector from lists of the indices',\n",
       "  'and values using the sparsevec function. You can also first create a sparse vector',\n",
       "  'of zeros (using spzeros(n)) and then assign values to the nonzero entries. A sparse',\n",
       "  'vector can be created from a non-sparse vector using sparse(x), which returns a',\n",
       "  'sparse version of x. nnz(x) gives the number of nonzero elements of a sparse vector.',\n",
       "  'Sparse vectors are overloaded to work as you imagine; for example, all the usual',\n",
       "  'vector operations work, and they are automatically recast as non-sparse vectors',\n",
       "  'when appropriate.',\n",
       "  '',\n",
       "  'julia> a = sparsevec( [ 123456, 123457 ], [ 1.0, -1.0 ], 10^6 )',\n",
       "  '',\n",
       "  '1000000-element SparseVector{Float64,Int64} with 2 stored entries:',\n",
       "  '',\n",
       "  '[123456 ] = 1.0',\n",
       "  '',\n",
       "  '[123457 ] = -1.0',\n",
       "  '',\n",
       "  'julia> length(a)',\n",
       "  '',\n",
       "  '1000000',\n",
       "  '',\n",
       "  'julia> nnz(a)',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  'julia> b = randn(10^6); # An ordinary (non-sparse) vector',\n",
       "  '',\n",
       "  'julia> @time 2*a; # Computed efficiently!',\n",
       "  '',\n",
       "  '0.000003 seconds (7 allocations: 384 bytes)',\n",
       "  '',\n",
       "  'julia> @time 2*b;',\n",
       "  '',\n",
       "  '0.003558 seconds (6 allocations: 7.630 MiB)',\n",
       "  '',\n",
       "  \"julia> @time a'*b; # Computed efficiently!\",\n",
       "  '0.000003 seconds (5 allocations: 176 bytes)',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '1.5 Complexity of vector computations 17',\n",
       "  '',\n",
       "  \"julia> @time b'*b;\",\n",
       "  '0.000450 seconds (5 allocations: 176 bytes)',\n",
       "  '',\n",
       "  'julia> @time c = a + b;',\n",
       "  '',\n",
       "  '0.002085 seconds (6 allocations: 7.630 MiB)',\n",
       "  '',\n",
       "  'In the last line, the sparse vector a is automatically converted to an ordinary vector',\n",
       "  '(array) so it can be added to the random vector; the result is a (non-sparse) vector',\n",
       "  'of length 106.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '18 1 Vectors',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Chapter 2',\n",
       "  '',\n",
       "  'Linear functions',\n",
       "  '',\n",
       "  '2.1 Linear functions',\n",
       "  '',\n",
       "  'Functions in Julia. Julia provides several methods for defining functions. A sim-',\n",
       "  'ple function given by an expression such as f(x) = x1 + x2 − x24 can be defined in',\n",
       "  'a single line.',\n",
       "  '',\n",
       "  'julia> f(x) = x[1] + x[2] - x[4]^2',\n",
       "  '',\n",
       "  'f (generic function with 1 method)',\n",
       "  '',\n",
       "  'julia> f([-1,0,1,2])',\n",
       "  '',\n",
       "  '-5',\n",
       "  '',\n",
       "  'Since the function definition refers to the first, second, and fourth elements of the',\n",
       "  'argument x, these have to be defined when you call or evaluate f(x); you’ll get an',\n",
       "  'error if, for example, x has dimension 3 or is a scalar.',\n",
       "  '',\n",
       "  'Superposition. Suppose a is an n-vector. The function f(x) = aTx is linear,',\n",
       "  'which means that for any n-vectors x and y, and any scalars α and β, the super-',\n",
       "  'position equality',\n",
       "  '',\n",
       "  'f(αx+ βy) = αf(x) + βf(y)',\n",
       "  '',\n",
       "  'holds. Superposition says that evaluating f at a linear combination of two vectors',\n",
       "  'is the same forming the linear combination of f evaluated at the two vectors.',\n",
       "  '',\n",
       "  'Let’s define the inner product function f for a specific value of a, and then',\n",
       "  'verify superposition in Julia for specific values of x, y, α, and β. (This check does',\n",
       "  'not show that the function is linear. It simply checks that superposition holds for',\n",
       "  'these specific values.)',\n",
       "  '',\n",
       "  'julia> a = [-2, 0, 1, -3];',\n",
       "  '',\n",
       "  \"julia> f(x) = a'*x # Inner product function\",\n",
       "  'f (generic function with 1 method)',\n",
       "  '',\n",
       "  'julia> x = [2, 2, -1, 1]; y= [0, 1, -1, 0];',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '20 2 Linear functions',\n",
       "  '',\n",
       "  'julia> alpha = 1.5; beta = -3.7;',\n",
       "  '',\n",
       "  'julia> lhs = f(alpha * x + beta * y)',\n",
       "  '',\n",
       "  '-8.3',\n",
       "  '',\n",
       "  'julia> rhs = alpha * f(x) + beta * f(y)',\n",
       "  '',\n",
       "  '-8.3',\n",
       "  '',\n",
       "  'For the function f(x) = aTx, we have f(e3) = a3. Let’s check that this holds in',\n",
       "  'our example.',\n",
       "  '',\n",
       "  'julia> e3 = [0, 0, 1, 0];',\n",
       "  '',\n",
       "  'julia> f(e3)',\n",
       "  '',\n",
       "  '1.0',\n",
       "  '',\n",
       "  'Examples. Let’s define the average function in Julia, and check its value for a',\n",
       "  'specific vector. (Julia’s Statistics package contains the average function, which',\n",
       "  'is called mean.)',\n",
       "  '',\n",
       "  \"julia> avg(x) = (ones(length(x)) / length(x))'*x;\",\n",
       "  'julia> x = [1, -3, 2, -1];',\n",
       "  '',\n",
       "  'julia> avg(x)',\n",
       "  '',\n",
       "  '-0.25',\n",
       "  '',\n",
       "  'The average function can be implemented more concisely as sum(x)/length(x).',\n",
       "  'The avg function is part of the VMLS package; once you install and then add this',\n",
       "  'package, you can use the avg function.',\n",
       "  '',\n",
       "  '2.2 Taylor approximation',\n",
       "  '',\n",
       "  'Taylor approximation. The (first-order) Taylor approximation of a function f :',\n",
       "  'Rn → R, at the point z, is the affine function of x given by',\n",
       "  '',\n",
       "  'f̂(x) = f(z) +∇f(z)T (x− z).',\n",
       "  '',\n",
       "  'For x near z, f̂(x) is very close to f(x). Let’s try a numerical example (see page 36)',\n",
       "  'using Julia.',\n",
       "  '',\n",
       "  'julia> f(x) = x[1] + exp(x[2]-x[1]); # A function',\n",
       "  '',\n",
       "  'julia> # And its gradient',\n",
       "  '',\n",
       "  'julia> grad_f(z) = [1-exp(z[2]-z[1]), exp(z[2]-z[1])];',\n",
       "  '',\n",
       "  'julia> z = [1, 2];',\n",
       "  '',\n",
       "  'julia> # Taylor approximation at z',\n",
       "  '',\n",
       "  \"julia> f_hat(x) = f(z) + grad_f(z)'*(x-z);\",\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.54',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '2.3 Regression model 21',\n",
       "  '',\n",
       "  'julia> # Let’s compare f and f_hat for some specific x’s',\n",
       "  '',\n",
       "  'julia> f([1,2]), f_hat([1,2])',\n",
       "  '',\n",
       "  '(3.718281828459045, 3.718281828459045)',\n",
       "  '',\n",
       "  'julia> f([0.96,1.98]), f_hat([0.96,1.98])',\n",
       "  '',\n",
       "  '(3.733194763964298, 3.732647465028226)',\n",
       "  '',\n",
       "  'julia> f([1.10,2.11]), f_hat([1.10,2.11])',\n",
       "  '',\n",
       "  '(3.845601015016916, 3.845464646743635)',\n",
       "  '',\n",
       "  '2.3 Regression model',\n",
       "  '',\n",
       "  'Regression model. The regression model is the affine function of x given by',\n",
       "  'f(x) = xTβ + v, where the n-vector β and the scalar v are the parameters in',\n",
       "  'the model. The regression model is used to guess or approximate a real or ob-',\n",
       "  'served value of the number y that is associated with x. (We’ll see later how to find',\n",
       "  'the parameters in a regression model, using data.)',\n",
       "  '',\n",
       "  'Let’s define the regression model for house sale prices described on page 39 of',\n",
       "  'VMLS, and compare its prediction to the true house sale price y for a few values',\n",
       "  'of x.',\n",
       "  '',\n",
       "  'julia> # Parameters in regression model',\n",
       "  '',\n",
       "  'julia> beta = [148.73, -18.85]; v = 54.40;',\n",
       "  '',\n",
       "  \"julia> y_hat(x) = x'*beta + v;\",\n",
       "  'julia> # Evaluate regression model prediction',\n",
       "  '',\n",
       "  'julia> x = [0.846, 1]; y = 115;',\n",
       "  '',\n",
       "  'julia> y_hat(x), y',\n",
       "  '',\n",
       "  '(161.37557999999999, 115)',\n",
       "  '',\n",
       "  'julia> x = [1.324,2]; y = 234.50;',\n",
       "  '',\n",
       "  'julia> y_hat(x), y',\n",
       "  '',\n",
       "  '(213.61852000000002, 234.5)',\n",
       "  '',\n",
       "  'Our first prediction is pretty bad; our second one is better.',\n",
       "  'A scatter plot of predicted and actual house prices (Figure 2.4 of VMLS) can',\n",
       "  '',\n",
       "  'be generated as follows. We use the VMLS function house_sales_data to obtain',\n",
       "  'the vectors price, area, beds (see appendix A).',\n",
       "  '',\n",
       "  'julia> D = house_sales_data();',\n",
       "  '',\n",
       "  'julia> price = D[\"price\"]',\n",
       "  '',\n",
       "  '774-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '94.905',\n",
       "  '',\n",
       "  '98.937',\n",
       "  '',\n",
       "  '100.309',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.56',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.2.4',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '22 2 Linear functions',\n",
       "  '',\n",
       "  '106.25',\n",
       "  '',\n",
       "  '107.502',\n",
       "  '',\n",
       "  '108.75',\n",
       "  '',\n",
       "  '110.7',\n",
       "  '',\n",
       "  '113.263',\n",
       "  '',\n",
       "  '116.25',\n",
       "  '',\n",
       "  '120.0',\n",
       "  '...',\n",
       "  '',\n",
       "  '229.027',\n",
       "  '',\n",
       "  '229.5',\n",
       "  '',\n",
       "  '230.0',\n",
       "  '',\n",
       "  '230.0',\n",
       "  '',\n",
       "  '232.425',\n",
       "  '',\n",
       "  '234.0',\n",
       "  '',\n",
       "  '235.0',\n",
       "  '',\n",
       "  '235.301',\n",
       "  '',\n",
       "  '235.738',\n",
       "  '',\n",
       "  'julia> area = D[\"area\"]',\n",
       "  '',\n",
       "  '774-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.941',\n",
       "  '',\n",
       "  '1.146',\n",
       "  '',\n",
       "  '0.909',\n",
       "  '',\n",
       "  '1.289',\n",
       "  '',\n",
       "  '1.02',\n",
       "  '',\n",
       "  '1.022',\n",
       "  '',\n",
       "  '1.134',\n",
       "  '',\n",
       "  '0.844',\n",
       "  '',\n",
       "  '0.795',\n",
       "  '',\n",
       "  '0.588',\n",
       "  '...',\n",
       "  '',\n",
       "  '1.358',\n",
       "  '',\n",
       "  '1.329',\n",
       "  '',\n",
       "  '1.715',\n",
       "  '',\n",
       "  '1.262',\n",
       "  '',\n",
       "  '2.28',\n",
       "  '',\n",
       "  '1.477',\n",
       "  '',\n",
       "  '1.216',\n",
       "  '',\n",
       "  '1.685',\n",
       "  '',\n",
       "  '1.362',\n",
       "  '',\n",
       "  'julia> beds = D[\"beds\"]',\n",
       "  '',\n",
       "  '774-element Array{Int64,1}:',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '2.3 Regression model 23',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  '3',\n",
       "  '',\n",
       "  '3',\n",
       "  '',\n",
       "  '3',\n",
       "  '',\n",
       "  '3',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  '2',\n",
       "  '...',\n",
       "  '',\n",
       "  '3',\n",
       "  '',\n",
       "  '4',\n",
       "  '',\n",
       "  '4',\n",
       "  '',\n",
       "  '3',\n",
       "  '',\n",
       "  '4',\n",
       "  '',\n",
       "  '3',\n",
       "  '',\n",
       "  '3',\n",
       "  '',\n",
       "  '4',\n",
       "  '',\n",
       "  '3',\n",
       "  '',\n",
       "  'julia> v = 54.4017;',\n",
       "  '',\n",
       "  'julia> beta = [ 147.7251, -18.8534 ];',\n",
       "  '',\n",
       "  'julia> predicted = v .+ beta[1] * area + beta[2] * beds;',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> scatter(price, predicted, lims = (0,800));',\n",
       "  '',\n",
       "  'julia> plot!([0, 800], [0, 800], linestyle = :dash);',\n",
       "  '',\n",
       "  'julia> # make axes equal and add labels',\n",
       "  '',\n",
       "  'julia> plot!(xlims = (0,800), ylims = (0,800), size = (500,500));',\n",
       "  '',\n",
       "  'julia> plot!(xlabel = \"Actual price\", ylabel = \"Predicted price\");',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '24 2 Linear functions',\n",
       "  '',\n",
       "  '0 200 400 600 800',\n",
       "  '0',\n",
       "  '',\n",
       "  '200',\n",
       "  '',\n",
       "  '400',\n",
       "  '',\n",
       "  '600',\n",
       "  '',\n",
       "  '800',\n",
       "  '',\n",
       "  'Actual price',\n",
       "  '',\n",
       "  'P',\n",
       "  're',\n",
       "  '',\n",
       "  'di',\n",
       "  'ct',\n",
       "  '',\n",
       "  'ed',\n",
       "  ' p',\n",
       "  '',\n",
       "  'ric',\n",
       "  'e',\n",
       "  '',\n",
       "  'Figure 2.1 Scatter plot of actual and predicted sale prices for 774 houses',\n",
       "  'sold in Sacramento during a five-day period.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Chapter 3',\n",
       "  '',\n",
       "  'Norm and distance',\n",
       "  '',\n",
       "  '3.1 Norm',\n",
       "  '',\n",
       "  'Norm. The norm ‖x‖ is written in Julia as norm(x). (It can be evaluated',\n",
       "  'several other ways too.) The norm function is contained in the Julia package',\n",
       "  'LinearAlgebra, so you must install and then add this package to use it; see page ix.',\n",
       "  '',\n",
       "  'julia> x = [ 2, -1, 2 ];',\n",
       "  '',\n",
       "  'julia> norm(x)',\n",
       "  '',\n",
       "  '3.0',\n",
       "  '',\n",
       "  \"julia> sqrt(x'*x))\",\n",
       "  '3.0',\n",
       "  '',\n",
       "  'julia> sqrt(sum(x.^2))',\n",
       "  '',\n",
       "  '3.0',\n",
       "  '',\n",
       "  'Triangle inequality. Let’s check the triangle inequality, ‖x+ y‖ ≤ ‖x‖+ ‖y‖, for',\n",
       "  'some specific values of x and y.',\n",
       "  '',\n",
       "  'julia> x = randn(10); y = randn(10);',\n",
       "  '',\n",
       "  'julia> lhs = norm(x+y)',\n",
       "  '',\n",
       "  '3.5830436972032644',\n",
       "  '',\n",
       "  'julia> rhs = norm(x) + norm(y)',\n",
       "  '',\n",
       "  '4.937368598697242',\n",
       "  '',\n",
       "  'RMS value. The RMS value of a vector x is rms(x) = ‖x‖/',\n",
       "  '√',\n",
       "  'n. In Julia, this is',\n",
       "  '',\n",
       "  'expressed as norm(x)/sqrt(length(x)). (The VMLS package contains this func-',\n",
       "  'tion, so you can use it once you’ve installed this package.)',\n",
       "  '',\n",
       "  'Let’s define a vector (which represents a signal, i.e., the value of some quantity',\n",
       "  'at uniformly space time instances), and find its RMS value. The following code',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '26 3 Norm and distance',\n",
       "  '',\n",
       "  '0.00 0.25 0.50 0.75 1.00',\n",
       "  '',\n",
       "  '-1',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  'Figure 3.1 A signal x. The horizontal lines show avg(x) + rms(x), avg(x),',\n",
       "  'and avg(x)− rms(x).',\n",
       "  '',\n",
       "  'plots the signal, its average value, and two constant signals at avg(x) ± rms(x)',\n",
       "  '(Figure 3.1).',\n",
       "  '',\n",
       "  'julia> rms(x) = norm(x) / sqrt(length(x));',\n",
       "  '',\n",
       "  'julia> t = 0:0.01:1; # List of times',\n",
       "  '',\n",
       "  'julia> x = cos.(8*t) - 2*sin.(11*t);',\n",
       "  '',\n",
       "  'julia> avg(x)',\n",
       "  '',\n",
       "  '-0.04252943783238692',\n",
       "  '',\n",
       "  'julia> rms(x)',\n",
       "  '',\n",
       "  '1.0837556422598',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> plot(t, x)',\n",
       "  '',\n",
       "  'julia> plot!(t, avg(x)*ones(length(x)))',\n",
       "  '',\n",
       "  'julia> plot!(t, (avg(x)+rms(x))*ones(length(x)), color = :green)',\n",
       "  '',\n",
       "  'julia> plot!(t, (avg(x)-rms(x))*ones(length(x)), color = :green)',\n",
       "  '',\n",
       "  'julia> plot!(legend = false)',\n",
       "  '',\n",
       "  'Chebyshev inequality. The Chebyshev inequality states that the number of entries',\n",
       "  'of an n-vector x that have absolute value at least a is no more than ‖x‖2/a2 =',\n",
       "  'n rms(x)2/a2. If this number is, say, 12.15, we can conclude that no more that',\n",
       "  '12 entries have absolute value at least a, since the number of entries is an integer.',\n",
       "  'So the Chebyshev bound can be improved to be floor(‖x‖2/a), where floor(u) is',\n",
       "  'the integer part of a positive number. Let’s define a function with the Chebyshev',\n",
       "  'bound, including the floor function improvement, and apply the bound to the signal',\n",
       "  'found above, for a specific value of a.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '3.2 Distance 27',\n",
       "  '',\n",
       "  'julia> # Define Chebyshev bound function',\n",
       "  '',\n",
       "  'julia> cheb_bound(x,a) = floor(norm(x)^2/a);',\n",
       "  '',\n",
       "  'julia> a = 1.5;',\n",
       "  '',\n",
       "  'julia> cheb_bound(x,a)',\n",
       "  '',\n",
       "  '79.0',\n",
       "  '',\n",
       "  'julia> # Number of entries of x with |x_i| >= a',\n",
       "  '',\n",
       "  'julia> sum(abs.(x) .>= a)',\n",
       "  '',\n",
       "  '20',\n",
       "  '',\n",
       "  'In the last line, the expression abs.(x) .>= a creates an array with entries that',\n",
       "  'are Boolean, i.e., true or false, depending on whether the corresponding entry of',\n",
       "  'x satisfies the inequality. When we sum the vector of Booleans, they are automat-',\n",
       "  'ically converted to (re-cast as) the numbers 1 and 0, respectively.',\n",
       "  '',\n",
       "  '3.2 Distance',\n",
       "  '',\n",
       "  'Distance. The distance between two vectors is dist(x, y) = ‖x − y‖. This is',\n",
       "  'written in Julia as norm(x-y). Let’s find the distance between the pairs of the',\n",
       "  'three vectors u, v, and w from page 49 of VMLS.',\n",
       "  '',\n",
       "  'julia> u = [1.8, 2.0, -3.7, 4.7];',\n",
       "  '',\n",
       "  'julia> v = [0.6, 2.1, 1.9, -1.4];',\n",
       "  '',\n",
       "  'julia> w = [2.0, 1.9, -4.0, 4.6];',\n",
       "  '',\n",
       "  'julia> norm(u-v), norm(u-w), norm(v-w)',\n",
       "  '',\n",
       "  '(8.36779540858881, 0.3872983346207417, 8.532877591996735)',\n",
       "  '',\n",
       "  'We can see that u and w are much closer to each other than u and v, or v and w.',\n",
       "  '',\n",
       "  'Nearest neighbor. We define a function that calculates the nearest neighbor of a',\n",
       "  'vector in a list of vectors, and try it on the points in Figure 3.3 of VMLS.',\n",
       "  '',\n",
       "  'julia> nearest_neighbor(x,z) = z[ argmin([norm(x-y) for y in z]) ];',\n",
       "  '',\n",
       "  'julia> z = ( [2,1], [7,2], [5.5,4], [4,8], [1,5], [9,6] );',\n",
       "  '',\n",
       "  'julia> nearest_neighbor([5,6], z)',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '5.5',\n",
       "  '',\n",
       "  '4.0',\n",
       "  '',\n",
       "  'julia> nearest_neighbor([3,3], z)',\n",
       "  '',\n",
       "  '2-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.3.1',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.3.3',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '28 3 Norm and distance',\n",
       "  '',\n",
       "  'On the first line, the expression [norm(x-y) for y in z] uses a convenient con-',\n",
       "  'struction in Julia. Here z is a list of vectors, and the expression expands to an array',\n",
       "  'with elements norm(x-z[1]), norm(x-z[2]), . . . . The function argmin applied to',\n",
       "  'this array returns the index of the smallest element.',\n",
       "  '',\n",
       "  'De-meaning a vector. We refer to the vector x − avg(x)1 as the de-meaned',\n",
       "  'version of x.',\n",
       "  '',\n",
       "  'julia> de_mean(x) = x .- avg(x); # Define de-mean function',\n",
       "  '',\n",
       "  'julia> x = [1, -2.2, 3];',\n",
       "  '',\n",
       "  'julia> avg(x)',\n",
       "  '',\n",
       "  '0.6',\n",
       "  '',\n",
       "  'julia> x_tilde = de_mean(x)',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.4',\n",
       "  '',\n",
       "  '-2.8',\n",
       "  '',\n",
       "  '2.4',\n",
       "  '',\n",
       "  'julia> avg(x_tilde)',\n",
       "  '',\n",
       "  '-1.4802973661668753e-16',\n",
       "  '',\n",
       "  '(The mean of x̃ is very very close to zero.)',\n",
       "  '',\n",
       "  '3.3 Standard deviation',\n",
       "  '',\n",
       "  'Standard deviation. We can define a function that corresponds to the VMLS',\n",
       "  'definition of the standard deviation of a vector, std(x) = ‖x − avg(x)1‖/',\n",
       "  '',\n",
       "  '√',\n",
       "  'n,',\n",
       "  '',\n",
       "  'where n is the length of the vector.',\n",
       "  '',\n",
       "  'julia> x = rand(100);',\n",
       "  '',\n",
       "  'julia> # VMLS definition of std',\n",
       "  '',\n",
       "  'julia> stdev(x) = norm(x-avg(x))/sqrt(length(x));',\n",
       "  '',\n",
       "  'julia> stdev(x)',\n",
       "  '',\n",
       "  '0.292205696281305',\n",
       "  '',\n",
       "  'This function is in the VMLS package, so you can use it once you’ve installed this',\n",
       "  'package. (Julia’s Statistics package has a similar function, std(x), which com-',\n",
       "  'putes the value ‖x− avg(x)1‖/',\n",
       "  '',\n",
       "  '√',\n",
       "  'n− 1, where n is the length of x.)',\n",
       "  '',\n",
       "  'Return and risk. We evaluate the mean return and risk (measured by standard',\n",
       "  'deviation) of the four time series Figure 3.4 of VMLS.',\n",
       "  '',\n",
       "  'julia> a = ones(10);',\n",
       "  '',\n",
       "  'julia> avg(a), stdev(a)',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.3.4',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '3.4 Angle 29',\n",
       "  '',\n",
       "  '(1.0, 0.0)',\n",
       "  '',\n",
       "  'julia> b = [ 5, 1, -2, 3, 6, 3, -1, 3, 4, 1 ];',\n",
       "  '',\n",
       "  'julia> avg(b), stdev(b)',\n",
       "  '',\n",
       "  '(2.3, 2.41039415863879)',\n",
       "  '',\n",
       "  'julia> c = [ 5, 7, -2, 2, -3, 1, -1, 2, 7, 8 ];',\n",
       "  '',\n",
       "  'julia> avg(c), stdev(c)',\n",
       "  '',\n",
       "  '(2.6, 3.773592452822641)',\n",
       "  '',\n",
       "  'julia> d = [ -1, -3, -4, -3, 7, -1, 0, 3, 9, 5 ];',\n",
       "  '',\n",
       "  'julia> avg(d), stdev(d)',\n",
       "  '',\n",
       "  '(1.2, 4.308131845707603)',\n",
       "  '',\n",
       "  'Standardizing a vector. If a vector x isn’t constant (i.e., at least two of its entries',\n",
       "  'are different), we can standardize it, by subtracting its mean and dividing by its',\n",
       "  'standard deviation. The resulting standardized vector has mean value zero and',\n",
       "  'RMS value one. Its entries are called z-scores. We’ll define a standardize function,',\n",
       "  'and then check it with a random vector.',\n",
       "  '',\n",
       "  'julia> function standardize(x)',\n",
       "  '',\n",
       "  'x_tilde = x .- avg(x) # De-meaned vector',\n",
       "  '',\n",
       "  'return x_tilde/rms(x_tilde)',\n",
       "  '',\n",
       "  'end',\n",
       "  '',\n",
       "  'julia> x = rand(100);',\n",
       "  '',\n",
       "  'julia> avg(x), rms(x)',\n",
       "  '',\n",
       "  '(0.510027255229345, 0.5883938729563185)',\n",
       "  '',\n",
       "  'julia> z = standardize(x);',\n",
       "  '',\n",
       "  'julia> avg(z), rms(z)',\n",
       "  '',\n",
       "  '(1.965094753586527e-16, 1.0)',\n",
       "  '',\n",
       "  'The mean or average value of the standarized vector z is very nearly zero.',\n",
       "  '',\n",
       "  '3.4 Angle',\n",
       "  '',\n",
       "  'Angle. Let’s define a function that computes the angle between two vectors. We',\n",
       "  'will call it ang because Julia already includes a function angle (for the phase angle',\n",
       "  'of a complex number).',\n",
       "  '',\n",
       "  'julia> # Define angle function, which returns radians',\n",
       "  '',\n",
       "  \"julia> ang(x,y) = acos(x'*y/(norm(x)*norm(y)));\",\n",
       "  'julia> a = [1,2,-1]; b=[2,0,-3];',\n",
       "  '',\n",
       "  'julia> ang(a,b)',\n",
       "  '',\n",
       "  '0.9689825515916383',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '30 3 Norm and distance',\n",
       "  '',\n",
       "  'julia> ang(a,b)*(360/(2*pi)) # Get angle in degrees',\n",
       "  '',\n",
       "  '55.51861062801842',\n",
       "  '',\n",
       "  'Correlation coefficient. The correlation coefficient between two vectors a and b',\n",
       "  '(with nonzero standard deviation) is defined as',\n",
       "  '',\n",
       "  'ρ =',\n",
       "  'ãT b̃',\n",
       "  '',\n",
       "  '‖ã‖‖b̃‖',\n",
       "  ',',\n",
       "  '',\n",
       "  'where ã and b̃ are the de-meaned versions of a and b, respectively. There is no',\n",
       "  'built-in function for correlation, so we can define one. We use function to calculate',\n",
       "  'the correlation coefficients of the three pairs of vectors in Figure 3.8 in VMLS.',\n",
       "  '',\n",
       "  'julia> function correl_coef(a,b)',\n",
       "  '',\n",
       "  'a_tilde = a .- avg(a)',\n",
       "  '',\n",
       "  'b_tilde = b .- avg(b)',\n",
       "  '',\n",
       "  \"return (a_tilde'*b_tilde)/(norm(a_tilde)*norm(b_tilde))\",\n",
       "  'end',\n",
       "  '',\n",
       "  'julia> a = [4.4, 9.4, 15.4, 12.4, 10.4, 1.4, -4.6, -5.6, -0.6, 7.4];',\n",
       "  '',\n",
       "  'julia> b = [6.2, 11.2, 14.2, 14.2, 8.2, 2.2, -3.8, -4.8, -1.8, 4.2];',\n",
       "  '',\n",
       "  'julia> correl_coef(a,b)',\n",
       "  '',\n",
       "  '0.9678196342570434',\n",
       "  '',\n",
       "  'julia> a = [4.1, 10.1, 15.1, 13.1, 7.1, 2.1, -2.9, -5.9, 0.1, 7.1];',\n",
       "  '',\n",
       "  'julia> b = [5.5, -0.5, -4.5, -3.5, 1.5, 7.5, 13.5, 14.5, 11.5, 4.5];',\n",
       "  '',\n",
       "  'julia> correl_coef(a,b)',\n",
       "  '',\n",
       "  '-0.9875211120643734',\n",
       "  '',\n",
       "  'julia> a = [-5.0, 0.0, 5.0, 8.0, 13.0, 11.0, 1.0, 6.0, 4.0, 7.0];',\n",
       "  '',\n",
       "  'julia> b = [5.8, 0.8, 7.8, 9.8, 0.8, 11.8, 10.8, 5.8, -0.2, -3.2];',\n",
       "  '',\n",
       "  'julia> correl_coef(a,b)',\n",
       "  '',\n",
       "  '0.004020976661367021',\n",
       "  '',\n",
       "  'The correlation coefficients of the three pairs of vectors are 96.8%, −98.8%, and',\n",
       "  '0.4%.',\n",
       "  '',\n",
       "  '3.5 Complexity',\n",
       "  '',\n",
       "  'Let’s check that the time to compute the correlation coefficient of two n-vectors is',\n",
       "  'approximately linear in n.',\n",
       "  '',\n",
       "  'julia> x = randn(10^6); y = randn(10^6);',\n",
       "  '',\n",
       "  'julia> @time correl_coef(x,y);',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.3.8',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '3.5 Complexity 31',\n",
       "  '',\n",
       "  '0.131375 seconds (33.01 k allocations: 16.913 MiB, 2.86% gc time)',\n",
       "  '',\n",
       "  'julia> @time correl_coef(x,y);',\n",
       "  '',\n",
       "  '0.023760 seconds (9 allocations: 15.259 MiB, 31.84% gc time)',\n",
       "  '',\n",
       "  'julia> x = randn(10^7); y = randn(10^7);',\n",
       "  '',\n",
       "  'julia> @time correl_coef(x,y)',\n",
       "  '',\n",
       "  '0.296075 seconds (9 allocations: 152.588 MiB, 30.16% gc time)',\n",
       "  '',\n",
       "  'julia> @time correl_coef(x,y)',\n",
       "  '',\n",
       "  '0.118979 seconds (9 allocations: 152.588 MiB, 21.53% gc time)',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '32 3 Norm and distance',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Chapter 4',\n",
       "  '',\n",
       "  'Clustering',\n",
       "  '',\n",
       "  '4.1 Clustering',\n",
       "  '',\n",
       "  '4.2 A clustering objective',\n",
       "  '',\n",
       "  'In Julia, we can store the list of vectors in a Julia list or tuple of N vectors. If',\n",
       "  'we call this list x, we can access the ith entry (which is a vector) using x[i]. To',\n",
       "  'specify the clusters or group membership, we can use a list of assignments called',\n",
       "  'assignment, where assignment[i] is the number of the group that vector x[i]',\n",
       "  'is assigned to. (This is an integer between 1 and k.) (In VMLS chapter 4, we',\n",
       "  'describe the assignments using a vector c or the subsets Gj .) We can store the',\n",
       "  'k cluster representatives as a Julia list called reps, with reps[j] the jth cluster',\n",
       "  'representative. (In VMLS we describe the representatives as the vectors z1, . . . , zk.)',\n",
       "  '',\n",
       "  'julia> Jclust(x,reps,assignment) =',\n",
       "  '',\n",
       "  'avg( [norm(x[i]-reps[assignment[i]])^2 for i=1:length(x)] )',\n",
       "  '',\n",
       "  'Jclust (generic function with 1 method)',\n",
       "  '',\n",
       "  'julia> x = [ [0,1], [1,0], [-1,1] ]',\n",
       "  '',\n",
       "  '3-element Array{Array{Int64,1},1}:',\n",
       "  '',\n",
       "  '[0, 1]',\n",
       "  '',\n",
       "  '[1, 0]',\n",
       "  '',\n",
       "  '[-1, 1]',\n",
       "  '',\n",
       "  'julia> reps = [ [1,1], [0,0] ]',\n",
       "  '',\n",
       "  '2-element Array{Array{Int64,1},1}:',\n",
       "  '',\n",
       "  '[1, 1]',\n",
       "  '',\n",
       "  '[0, 0]',\n",
       "  '',\n",
       "  'julia> assignment = [1,2,1]',\n",
       "  '',\n",
       "  '3-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#chapter.4',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '34 4 Clustering',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  'julia> Jclust(x,reps,assignment)',\n",
       "  '',\n",
       "  '2.0',\n",
       "  '',\n",
       "  'julia> assignment = [1,1,2]',\n",
       "  '',\n",
       "  '1.3333333333333333',\n",
       "  '',\n",
       "  '4.3 The k-means algorithm',\n",
       "  '',\n",
       "  'We write a simple Julia implementation of the k-means algorithm and apply it to',\n",
       "  'a set of points in a plane, similar to the example in Figure 4.1 of VMLS.',\n",
       "  '',\n",
       "  'We first create a function kmeans that can be called as',\n",
       "  '',\n",
       "  'julia> assignment, representatives = kmeans(x, k)',\n",
       "  '',\n",
       "  'where x is an array of N vectors and k is the number of groups. The first output',\n",
       "  'argument is an array of N integers, containing the computed group assignments',\n",
       "  '(integers from 1 to k). The second output argument is an array of k vectors, with',\n",
       "  'the k group representatives. We also include two optional keyword arguments, with',\n",
       "  'a limit on the number of iterations and a tolerance used in the stopping condition.',\n",
       "  '',\n",
       "  '1 function kmeans(x, k; maxiters = 100, tol = 1e-5)',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  '3 N = length(x)',\n",
       "  '',\n",
       "  '4 n = length(x[1])',\n",
       "  '',\n",
       "  '5 distances = zeros(N) # used to store the distance of each',\n",
       "  '',\n",
       "  '6 # point to the nearest representative.',\n",
       "  '',\n",
       "  '7 reps = [zeros(n) for j=1:k] # used to store representatives.',\n",
       "  '',\n",
       "  '8',\n",
       "  '',\n",
       "  '9 # ’assignment’ is an array of N integers between 1 and k.',\n",
       "  '',\n",
       "  '10 # The initial assignment is chosen randomly.',\n",
       "  '',\n",
       "  '11 assignment = [ rand(1:k) for i in 1:N ]',\n",
       "  '',\n",
       "  '12',\n",
       "  '',\n",
       "  '13 Jprevious = Inf # used in stopping condition',\n",
       "  '',\n",
       "  '14 for iter = 1:maxiters',\n",
       "  '',\n",
       "  '15',\n",
       "  '',\n",
       "  '16 # Cluster j representative is average of points in cluster j.',\n",
       "  '',\n",
       "  '17 for j = 1:k',\n",
       "  '',\n",
       "  '18 group = [i for i=1:N if assignment[i] == j]',\n",
       "  '',\n",
       "  '19 reps[j] = sum(x[group]) / length(group);',\n",
       "  '',\n",
       "  '20 end;',\n",
       "  '',\n",
       "  '21',\n",
       "  '',\n",
       "  '22 # For each x[i], find distance to the nearest representative',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.4.1',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '4.3 The k-means algorithm 35',\n",
       "  '',\n",
       "  '23 # and its group index.',\n",
       "  '',\n",
       "  '24 for i = 1:N',\n",
       "  '',\n",
       "  '25 (distances[i], assignment[i]) =',\n",
       "  '',\n",
       "  '26 findmin([norm(x[i] - reps[j]) for j = 1:k])',\n",
       "  '',\n",
       "  '27 end;',\n",
       "  '',\n",
       "  '28',\n",
       "  '',\n",
       "  '29 # Compute clustering objective.',\n",
       "  '',\n",
       "  '30 J = norm(distances)^2 / N',\n",
       "  '',\n",
       "  '31',\n",
       "  '',\n",
       "  '32 # Show progress and terminate if J stopped decreasing.',\n",
       "  '',\n",
       "  '33 println(\"Iteration \", iter, \": Jclust = \", J, \".\")',\n",
       "  '',\n",
       "  '34 if iter > 1 && abs(J - Jprevious) < tol * J',\n",
       "  '',\n",
       "  '35 return assignment, reps',\n",
       "  '',\n",
       "  '36 end',\n",
       "  '',\n",
       "  '37 Jprevious = J',\n",
       "  '',\n",
       "  '38 end',\n",
       "  '',\n",
       "  '39',\n",
       "  '',\n",
       "  '40 end',\n",
       "  '',\n",
       "  'Initialization. As discussed in VMLS (page 76), the k-means algorithm can start',\n",
       "  'from a random initial choice of representatives, or from a random assignment of',\n",
       "  'the points in k groups. In this implementation, we use the second option (line 11).',\n",
       "  'The Julia function rand(1:k) picks a random number from the set 1:k, i.e., the',\n",
       "  'integers 1, . . . , k. On line 11 we create an array assignment of N elements, with',\n",
       "  'each element chosen by calling rand(1:k).',\n",
       "  '',\n",
       "  'Updating group representatives. Lines 17–20 update the k group representa-',\n",
       "  'tives. In line 18, we find the indexes of the points in cluster j and collect them',\n",
       "  'in an array group. The expression x[group] on line 19 constructs an array from',\n",
       "  'the subset of elements of x indexed by group. The function sum computes the',\n",
       "  'sum of the elements of the array x[group]. Dividing by the number of elements',\n",
       "  'length(x[group]) gives the average of the vectors in the group. The result is jth',\n",
       "  'the group representative. This vector is stored as the jth element in an array reps',\n",
       "  'of length N .',\n",
       "  '',\n",
       "  'Updating group assignments. On lines 24–27 we update the group assignments.',\n",
       "  'The Julia function findmin computes both the minimum of a sequence of numbers',\n",
       "  'and the position of the minimum in the sequence. The result is returned as a 2-',\n",
       "  'tuple. On lines 25–26, we apply findmin to the array of k distances of point x[i]',\n",
       "  'to the k representatives. We store the distance to the nearest representative in',\n",
       "  'distances[i], and the index of the nearest representative (i.e., the new assignment',\n",
       "  'of point i) in assignment[i].',\n",
       "  '',\n",
       "  'Clustering objective. On line 30 we compute the clustering objective Jclust (equa-',\n",
       "  'tion (4.1) in VMLS) as the square of the RMS value of the vector of distances.',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.91',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#equation.4.2.1',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '36 4 Clustering',\n",
       "  '',\n",
       "  '-1 0 1 2',\n",
       "  '-2',\n",
       "  '',\n",
       "  '-1',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  'Figure 4.1 300 points in a plane.',\n",
       "  '',\n",
       "  'Convergence. We terminate the algorithm when the improvement in the cluster-',\n",
       "  'ing objective becomes very small (lines 34–36).',\n",
       "  '',\n",
       "  'Example. We apply the algorithm on a randomly generated set of N = 300 points,',\n",
       "  'shown in Figure 4.1. These points were generated as follows.',\n",
       "  '',\n",
       "  'julia> X = vcat( [ 0.3*randn(2) for i = 1:100 ],',\n",
       "  '',\n",
       "  '[ [1,1] + 0.3*randn(2) for i = 1:100 ],',\n",
       "  '',\n",
       "  '[ [1,-1] + 0.3*randn(2) for i = 1:100 ] )',\n",
       "  '',\n",
       "  'julia> scatter([x[1] for x in X], [x[2] for x in X])',\n",
       "  '',\n",
       "  'julia> plot!(legend = false, grid = false, size = (500,500),',\n",
       "  '',\n",
       "  'xlims = (-1.5,2.5), ylims = (-2,2))',\n",
       "  '',\n",
       "  'On the first line we generate three arrays of vectors. Each set consists of 100',\n",
       "  'vectors chosen randomly around one of the three points (0, 0), (1, 1), and (1,−1).',\n",
       "  'The three arrays are concatenated using vcat to get an array of 300 points.',\n",
       "  '',\n",
       "  'Next, we apply our kmeans function and make a figure with the three clusters',\n",
       "  '(Figure 4.2).',\n",
       "  '',\n",
       "  'julia> assignment, reps = kmeans(X, 3)',\n",
       "  '',\n",
       "  'Iteration 1: Jclust = 0.8815722022603146.',\n",
       "  '',\n",
       "  'Iteration 2: Jclust = 0.24189035975341422.',\n",
       "  '',\n",
       "  'Iteration 3: Jclust = 0.18259342207994636.',\n",
       "  '',\n",
       "  'Iteration 4: Jclust = 0.1800980527878161.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '4.4 Examples 37',\n",
       "  '',\n",
       "  '-1 0 1 2',\n",
       "  '-2',\n",
       "  '',\n",
       "  '-1',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  'Figure 4.2 Final clustering.',\n",
       "  '',\n",
       "  'Iteration 5: Jclust = 0.17993051934500726.',\n",
       "  '',\n",
       "  'Iteration 6: Jclust = 0.17988967509836415.',\n",
       "  '',\n",
       "  'Iteration 7: Jclust = 0.17988967509836415.',\n",
       "  '',\n",
       "  'julia> grps = [[X[i] for i=1:N if assignment[i] == j] for j=1:k]',\n",
       "  '',\n",
       "  'julia> scatter([c[1] for c in grps[1]], [c[2] for c in grps[1]])',\n",
       "  '',\n",
       "  'julia> scatter!([c[1] for c in grps[2]], [c[2] for c in grps[2]])',\n",
       "  '',\n",
       "  'julia> scatter!([c[1] for c in grps[3]], [c[2] for c in grps[3]])',\n",
       "  '',\n",
       "  'julia> plot!(legend = false, grid = false, size = (500,500),',\n",
       "  '',\n",
       "  'xlims = (-1.5,2.5), ylims = (-2,2))',\n",
       "  '',\n",
       "  '4.4 Examples',\n",
       "  '',\n",
       "  '4.5 Applications',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '38 4 Clustering',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Chapter 5',\n",
       "  '',\n",
       "  'Linear independence',\n",
       "  '',\n",
       "  '5.1 Linear dependence',\n",
       "  '',\n",
       "  '5.2 Basis',\n",
       "  '',\n",
       "  'Cash flow replication. Let’s consider cash flows over 3 periods, given by 3-vectors.',\n",
       "  'We know from VMLS page 93 that the vectors',\n",
       "  '',\n",
       "  'e1 =',\n",
       "  '',\n",
       "  '\\uf8ee\\uf8f0 10',\n",
       "  '0',\n",
       "  '',\n",
       "  '\\uf8f9\\uf8fb , l1 =',\n",
       "  '\\uf8ee\\uf8f0 1−(1 + r)',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  '\\uf8f9\\uf8fb , l2 =',\n",
       "  '\\uf8ee\\uf8f0 01',\n",
       "  '−(1 + r)',\n",
       "  '',\n",
       "  '\\uf8f9\\uf8fb',\n",
       "  'form a basis, wher r is the (positive) per-period interest rate. The first vector e1',\n",
       "  'is a single payment of $1 in period (time) t = 1. The second vector l1 is loan of $1',\n",
       "  'in period t = 1, paid back in period t = 2 with interest r. The third vector l2 is',\n",
       "  'loan of $1 in period t = 2, paid back in period t = 3 with interest r. Let’s use this',\n",
       "  'basis to replicate the cash flow c = (1, 2,−3) as',\n",
       "  '',\n",
       "  'c = α1e1 + α2l1 + α3l2 = α1',\n",
       "  '',\n",
       "  '\\uf8ee\\uf8f0 10',\n",
       "  '0',\n",
       "  '',\n",
       "  '\\uf8f9\\uf8fb+ α2',\n",
       "  '\\uf8ee\\uf8f0 1−(1 + r)',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  '\\uf8f9\\uf8fb+ α3',\n",
       "  '\\uf8ee\\uf8f0 01',\n",
       "  '−(1 + r)',\n",
       "  '',\n",
       "  '\\uf8f9\\uf8fb .',\n",
       "  'From the third component we have c3 = α3(−(1 + r)), so α3 = −c3/(1 + r). From',\n",
       "  'the second component we have',\n",
       "  '',\n",
       "  'c2 = α2(−(1 + r)) + α3 = α2(−(1 + r))− c3/(1 + r),',\n",
       "  '',\n",
       "  'so α2 = −c2/(1 + r)− c3/(1 + r)2. Finally from c1 = α1 + α2, we have',\n",
       "  '',\n",
       "  'α1 = c1 + c2/(1 + r) + c3/(1 + r)',\n",
       "  '2,',\n",
       "  '',\n",
       "  'which is the net present value (NPV) of the cash flow c.',\n",
       "  'Let’s check this in Julia using an interest rate of 5% per period, and the specific',\n",
       "  '',\n",
       "  'cash flow c = (1, 2,−3).',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.108',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '40 5 Linear independence',\n",
       "  '',\n",
       "  'julia> r = 0.05;',\n",
       "  '',\n",
       "  'julia> e1 = [1,0,0]; l1 = [1,-(1+r),0]; l2 = [0,1,-(1+r)];',\n",
       "  '',\n",
       "  'julia> c = [1,2,-3];',\n",
       "  '',\n",
       "  'julia> # Coefficients of expansion',\n",
       "  '',\n",
       "  'julia> alpha3 = -c[3]/(1+r);',\n",
       "  '',\n",
       "  'julia> alpha2 = -c[2]/(1+r) -c[3]/(1+r)^2;',\n",
       "  '',\n",
       "  'julia> alpha1 = c[1] + c[2]/(1+r) + c[3]/(1+r)^2 # NPV of cash flow',\n",
       "  '',\n",
       "  '0.18367346938775508',\n",
       "  '',\n",
       "  'julia> alpha1*e1 + alpha2*l1 + alpha3*l2',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '1.0',\n",
       "  '',\n",
       "  '2.0',\n",
       "  '',\n",
       "  '-3.0',\n",
       "  '',\n",
       "  '(Later in the course we’ll an automated and simple way to find the coefficients in',\n",
       "  'the expansion of a vector in a basis.)',\n",
       "  '',\n",
       "  '5.3 Orthonormal vectors',\n",
       "  '',\n",
       "  'Expansion in an orthonormal basis. Let’s check that the vectors',\n",
       "  '',\n",
       "  'a1 =',\n",
       "  '',\n",
       "  '\\uf8ee\\uf8f0 00',\n",
       "  '−1',\n",
       "  '',\n",
       "  '\\uf8f9\\uf8fb , a2 = 1√',\n",
       "  '2',\n",
       "  '',\n",
       "  '\\uf8ee\\uf8f0 11',\n",
       "  '0',\n",
       "  '',\n",
       "  '\\uf8f9\\uf8fb , a3 = 1√',\n",
       "  '2',\n",
       "  '',\n",
       "  '\\uf8ee\\uf8f0 1−1',\n",
       "  '0',\n",
       "  '',\n",
       "  '\\uf8f9\\uf8fb ,',\n",
       "  'form an orthonormal basis, and check the expansion of x = (1, 2, 3) in this basis,',\n",
       "  '',\n",
       "  'x = (aT1 x)a1 + · · ·+ (aTnx)an.',\n",
       "  '',\n",
       "  'julia> a1 = [0,0,-1]; a2 = [1,1,0]/sqrt(2); a3 = [1,-1,0]/sqrt(2);',\n",
       "  '',\n",
       "  'julia> norm(a1), norm(a2), norm(a3)',\n",
       "  '',\n",
       "  '(1.0, 0.9999999999999999, 0.9999999999999999)',\n",
       "  '',\n",
       "  \"julia> a1'*a2, a1'*a3, a2'*a3\",\n",
       "  '(0.0, 0.0, 0.0)',\n",
       "  '',\n",
       "  'julia> x = [1,2,3]',\n",
       "  '',\n",
       "  '3-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  '3',\n",
       "  '',\n",
       "  'julia> # Get coefficients of x in orthonormal basis',\n",
       "  '',\n",
       "  \"julia> beta1 = a1'*x; beta2 = a2'*x; beta3 = a3'*x;\",\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '5.4 Gram–Schmidt algorithm 41',\n",
       "  '',\n",
       "  'julia> # Expansion of x in basis',\n",
       "  '',\n",
       "  'julia> xexp = beta1*a1 + beta2*a2 + beta3*a3',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '1.0',\n",
       "  '',\n",
       "  '2.0',\n",
       "  '',\n",
       "  '3.0',\n",
       "  '',\n",
       "  '5.4 Gram–Schmidt algorithm',\n",
       "  '',\n",
       "  'The following is a Julia implementation of Algorithm 5.1 in VMLS (Gram–Schmidt',\n",
       "  'algorithm). It takes as input an array [ a[1], a[2], ..., a[k] ], containing',\n",
       "  'the k vectors a1, . . . , ak. If the vectors are linearly independent, it returns an array',\n",
       "  '[ q[1], ..., q[k] ] with the orthonormal set of vectors computed by the Gram–',\n",
       "  'Schmidt algorithm. If the vectors are linearly dependent and the Gram–Schnidt',\n",
       "  'algorithm terminates early in iteration i, it returns the array [ q[1], ..., q[i] ]',\n",
       "  'of length i.',\n",
       "  '',\n",
       "  '1 function gram_schmidt(a; tol = 1e-10)',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  '3 q = []',\n",
       "  '',\n",
       "  '4 for i = 1:length(a)',\n",
       "  '',\n",
       "  '5 qtilde = a[i]',\n",
       "  '',\n",
       "  '6 for j = 1:i-1',\n",
       "  '',\n",
       "  \"7 qtilde -= (q[j]'*a[i]) * q[j]\",\n",
       "  '8 end',\n",
       "  '',\n",
       "  '9 if norm(qtilde) < tol',\n",
       "  '',\n",
       "  '10 println(\"Vectors are linearly dependent.\")',\n",
       "  '',\n",
       "  '11 return q',\n",
       "  '',\n",
       "  '12 end',\n",
       "  '',\n",
       "  '13 push!(q, qtilde/norm(qtilde))',\n",
       "  '',\n",
       "  '14 end;',\n",
       "  '',\n",
       "  '15 return q',\n",
       "  '',\n",
       "  '16 end',\n",
       "  '',\n",
       "  'On line 3, we initialize the output array as the empty array. In each iteration, we',\n",
       "  'add the next vector to the array using the push! function (line 13).',\n",
       "  '',\n",
       "  'Example. We apply the function to the example on page 100 of VMLS.',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#algorithmctr.5.1',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.117',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '42 5 Linear independence',\n",
       "  '',\n",
       "  'julia> a = [ [-1, 1, -1, 1], [-1, 3, -1, 3], [1, 3, 5, 7] ]',\n",
       "  '',\n",
       "  '3-element Array{Array{Int64,1},1}:',\n",
       "  '',\n",
       "  '[-1, 1, -1, 1]',\n",
       "  '',\n",
       "  '[-1, 3, -1, 3]',\n",
       "  '',\n",
       "  '[1, 3, 5, 7]',\n",
       "  '',\n",
       "  'julia> q = gram_schmidt(a)',\n",
       "  '',\n",
       "  '3-element Array{Any,1}:',\n",
       "  '',\n",
       "  '[-0.5, 0.5, -0.5, 0.5]',\n",
       "  '',\n",
       "  '[0.5, 0.5, 0.5, 0.5]',\n",
       "  '',\n",
       "  '[-0.5, -0.5, 0.5, 0.5]',\n",
       "  '',\n",
       "  'julia> # test orthnormality',\n",
       "  '',\n",
       "  'julia> norm(q[1])',\n",
       "  '',\n",
       "  '1.0',\n",
       "  '',\n",
       "  \"julia> q[1]'*q[2]\",\n",
       "  '0.0',\n",
       "  '',\n",
       "  \"julia> q[1]'*q[3]\",\n",
       "  '0.0',\n",
       "  '',\n",
       "  'julia> norm(q[2])',\n",
       "  '',\n",
       "  '1.0',\n",
       "  '',\n",
       "  \"julia> q[2]'*q[3]\",\n",
       "  '0.0',\n",
       "  '',\n",
       "  'julia> norm(q[3])',\n",
       "  '',\n",
       "  '1.0',\n",
       "  '',\n",
       "  'Example of early termination. If we replace a3 with a linear combination of a1',\n",
       "  'and a2 the set becomes linearly dependent.',\n",
       "  '',\n",
       "  'julia> b = [ a[1], a[2], 1.3*a[1] + 0.5*a[2] ]',\n",
       "  '',\n",
       "  '3-element Array{Array{Float64,1},1}:',\n",
       "  '',\n",
       "  '[-1.0, 1.0, -1.0, 1.0]',\n",
       "  '',\n",
       "  '[-1.0, 3.0, -1.0, 3.0]',\n",
       "  '',\n",
       "  '[-1.8, 2.8, -1.8, 2.8]',\n",
       "  '',\n",
       "  'julia> q = gram_schmidt(b)',\n",
       "  '',\n",
       "  'Vectors are linearly dependent.',\n",
       "  '',\n",
       "  '2-element Array{Any,1}:',\n",
       "  '',\n",
       "  '[-0.5, 0.5, -0.5, 0.5]',\n",
       "  '',\n",
       "  '[0.5, 0.5, 0.5, 0.5]',\n",
       "  '',\n",
       "  'Example of independence-dimension inequality. We know that any three 2-',\n",
       "  'vectors must be dependent. Let’s use the Gram-Schmidt algorithm to verify this',\n",
       "  'for three specific vectors.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '5.4 Gram–Schmidt algorithm 43',\n",
       "  '',\n",
       "  'julia> three_two_vectors = [ [1,1], [1,2], [-1,1] ]',\n",
       "  '',\n",
       "  'julia> q = gram_schmidt(three_two_vectors)',\n",
       "  '',\n",
       "  'Vectors are linearly dependent.',\n",
       "  '',\n",
       "  '2-element Array{Any,1}:',\n",
       "  '',\n",
       "  '[0.707107, 0.707107]',\n",
       "  '',\n",
       "  '[-0.707107, 0.707107]',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '44 5 Linear independence',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Chapter 6',\n",
       "  '',\n",
       "  'Matrices',\n",
       "  '',\n",
       "  '6.1 Matrices',\n",
       "  '',\n",
       "  'Creating matrices from the entries. Matrices are represented in Julia as 2-',\n",
       "  'dimensional arrays. These are constructed by giving the elements in each row,',\n",
       "  'separated by space, with the rows separated by semicolons. For example, the 3× 4',\n",
       "  'matrix',\n",
       "  '',\n",
       "  'A =',\n",
       "  '',\n",
       "  '\\uf8ee\\uf8f0 0 1 −2.3 0.11.3 4 −0.1 0',\n",
       "  '4.1 −1 0 1.7',\n",
       "  '',\n",
       "  '\\uf8f9\\uf8fb',\n",
       "  'is constructed in Julia as',\n",
       "  '',\n",
       "  'julia> A = [0.0 1.0 -2.3 0.1;',\n",
       "  '',\n",
       "  '1.3 4.0 -0.1 0.0;',\n",
       "  '',\n",
       "  '4.1 -1.0 0.0 1.7]',\n",
       "  '',\n",
       "  '3×4 Array{Float64,2}:',\n",
       "  '0.0 1.0 -2.3 0.1',\n",
       "  '',\n",
       "  '1.3 4.0 -0.1 0.0',\n",
       "  '',\n",
       "  '4.1 -1.0 0.0 1.7',\n",
       "  '',\n",
       "  '(Here, Array{Float64,2} above the array tells us that the array is 2-dimensional,',\n",
       "  'and its entries are 64-bit floating-point numbers.) In this example, we put the',\n",
       "  'different rows of the matrix on different lines, which makes the code more readable,',\n",
       "  'but there is no need to do this; we get the same matrix with',\n",
       "  '',\n",
       "  'julia> A = [0 1 -2.3 0.1; 1.3 4 -0.1 0; 4.1 -1 0 1.7]',\n",
       "  '',\n",
       "  '3×4 Array{Float64,2}:',\n",
       "  '0.0 1.0 -2.3 0.1',\n",
       "  '',\n",
       "  '1.3 4.0 -0.1 0.0',\n",
       "  '',\n",
       "  '4.1 -1.0 0.0 1.7',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '46 6 Matrices',\n",
       "  '',\n",
       "  'The Julia function size(A) gives the size, as a tuple. It can also be called as',\n",
       "  'size(A,1) or size(A,2) to get only the number of rows or columns. As an exam-',\n",
       "  'ple, we create a function that determines if a matrix is tall.',\n",
       "  '',\n",
       "  'julia> m, n = size(A)',\n",
       "  '',\n",
       "  '(3, 4)',\n",
       "  '',\n",
       "  'julia> m',\n",
       "  '',\n",
       "  '3',\n",
       "  '',\n",
       "  'julia> n',\n",
       "  '',\n",
       "  '4',\n",
       "  '',\n",
       "  'julia> size(A,1)',\n",
       "  '',\n",
       "  '3',\n",
       "  '',\n",
       "  'julia> size(A,2)',\n",
       "  '',\n",
       "  '4',\n",
       "  '',\n",
       "  'julia> tall(X) = size(X,1)>size(X,2);',\n",
       "  '',\n",
       "  'julia> tall(A)',\n",
       "  '',\n",
       "  'false',\n",
       "  '',\n",
       "  'In the function definition, the number of rows and the number of columns are',\n",
       "  'combined using the relational operator <, which gives a Boolean.',\n",
       "  '',\n",
       "  'Indexing entries. We get the i, j entry of a matrix A using A[i,j]. We can also',\n",
       "  'assign a new value to an entry.',\n",
       "  '',\n",
       "  'julia> A[2,3] # Get 2,3 entry of A',\n",
       "  '',\n",
       "  '-0.1',\n",
       "  '',\n",
       "  'julia> A[1,3] = 7.5; # Set 1,3 entry of A to 7.5',\n",
       "  '',\n",
       "  '7.5',\n",
       "  '',\n",
       "  'julia> A',\n",
       "  '',\n",
       "  '3×4 Array{Float64,2}:',\n",
       "  '0.0 1.0 7.5 0.1',\n",
       "  '',\n",
       "  '1.3 4.0 -0.1 0.0',\n",
       "  '',\n",
       "  '4.1 -1.0 0.0 1.7',\n",
       "  '',\n",
       "  'Single index indexing. Julia allows you to access an entry of a matrix using',\n",
       "  'only one index. To use this, you need to know that matrices in Julia are stored',\n",
       "  'in column-major order. This means that a matrix can be considered as a one-',\n",
       "  'dimensional array, with the first column stacked on top of the second, stacked on',\n",
       "  'top of the third, and so on. For example, the elements of the matrix',\n",
       "  '',\n",
       "  'Z =',\n",
       "  '',\n",
       "  '[',\n",
       "  '−1 0 2',\n",
       "  '−1 2 3',\n",
       "  '',\n",
       "  ']',\n",
       "  'are stored in the order',\n",
       "  '',\n",
       "  '−1, −1, 0, 2, 2, 3.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '6.1 Matrices 47',\n",
       "  '',\n",
       "  'With single index indexing, Z[5] is the fifth element in this sequence.',\n",
       "  '',\n",
       "  'julia> Z = [ -1 0 2; -1 2 -3];',\n",
       "  '',\n",
       "  'julia> Z[5]',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  'This is very much not standard mathematical notation, and we would never use',\n",
       "  'this in VMLS. But it can be handy in some cases when you are using Julia.',\n",
       "  '',\n",
       "  'Equality of matrices. A == B determines whether the matrices A and B are equal.',\n",
       "  'The expression A .== B creates a matrix whose entries are Boolean, depending',\n",
       "  'on whether the corresponding entries of A and B are the same. The expression',\n",
       "  'sum(A .== B) gives the number of entries of A and B that are equal.',\n",
       "  '',\n",
       "  'julia> B = copy(A);',\n",
       "  '',\n",
       "  'julia> B[2,2] = 0;',\n",
       "  '',\n",
       "  'julia> A == B',\n",
       "  '',\n",
       "  'false',\n",
       "  '',\n",
       "  'julia> A .== B',\n",
       "  '',\n",
       "  '3×4 BitArray{2}:',\n",
       "  'true true true true',\n",
       "  '',\n",
       "  'true false true true',\n",
       "  '',\n",
       "  'true true true true',\n",
       "  '',\n",
       "  'julia> sum(A .== B)',\n",
       "  '',\n",
       "  '11',\n",
       "  '',\n",
       "  'Row and column vectors. In Julia, as in VMLS, n-vectors are the same as n× 1',\n",
       "  'matrices.',\n",
       "  '',\n",
       "  'julia> a = [ -2.1 -3 0 ] # A 3-row vector or 1x3 matrix',\n",
       "  '',\n",
       "  '1×3 Array{Float64,2}:',\n",
       "  '-2.1 -3.0 0.0',\n",
       "  '',\n",
       "  'julia> b = [ -2.1; -3; 0 ] # A 3-vector or 3x1 matrix',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '-2.1',\n",
       "  '',\n",
       "  '-3.0',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  'The output reveals a small subtlety that generally won’t affect you. You can see',\n",
       "  'that b has type Array{Float64,1}. The final 1 means that it is a 1D array;',\n",
       "  'size(b) gives (3,), whereas you might think it would or should be (3,1). So you',\n",
       "  'might say that in Julia, n × 1 matrices are n-vectors. This is why we say above',\n",
       "  'that n-vectors and n× 1 matrices are almost the same in Julia.',\n",
       "  '',\n",
       "  'Slicing and submatrices. Using colon notation you can extract a submatrix.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '48 6 Matrices',\n",
       "  '',\n",
       "  'julia> A = [ -1 0 1 0 ; 2 -3 0 1 ; 0 4 -2 1]',\n",
       "  '',\n",
       "  '3×4 Array{Int64,2}:',\n",
       "  '-1 0 1 0',\n",
       "  '',\n",
       "  '2 -3 0 1',\n",
       "  '',\n",
       "  '0 4 -2 1',\n",
       "  '',\n",
       "  'julia> A[1:2,3:4]',\n",
       "  '',\n",
       "  '2×2 Array{Float64,2}:',\n",
       "  '1 0',\n",
       "  '',\n",
       "  '0 1',\n",
       "  '',\n",
       "  'This is very similar to the mathematical notation in VMLS, where this submatrix',\n",
       "  'would be denoted A1:2,3:4. You can also assign a submatrix using slicing (index',\n",
       "  'range) notation.',\n",
       "  '',\n",
       "  'A very useful shortcut is the index range : which refers to the whole index range',\n",
       "  'for that index. This can be used to extract the rows and columns of a matrix.',\n",
       "  '',\n",
       "  'julia> A[:,3] # Third column of A',\n",
       "  '',\n",
       "  '3-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  '-2',\n",
       "  '',\n",
       "  'julia> A[2,:] # Second row of A, returned as column vector!',\n",
       "  '',\n",
       "  '4-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  '-3',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  'In mathematical (VMLS) notation, we say that A[2,:] returns the transpose of',\n",
       "  'the second row of A.',\n",
       "  '',\n",
       "  'As with vectors, Julia’s slicing and selection is not limited to contiguous ranges',\n",
       "  'of indexes. For example, we can reverse the order of the rows of a matrix X using',\n",
       "  '',\n",
       "  'julia> m = size(X,1)',\n",
       "  '',\n",
       "  'julia> X[m:-1:1,:] # Matrix X with row order reversed',\n",
       "  '',\n",
       "  'Julia’s single indexing for matrices can be used with index ranges or sets. For',\n",
       "  'example if X is an m × n matrix, X[:] is a vector of size mn that consists of the',\n",
       "  'columns of X stacked on top of each other. The Julia function reshape(X,(k,l))',\n",
       "  'gives a new k× l matrix, with the entries taken in the column-major order from X.',\n",
       "  '(We must have mn = kl, i.e., the original and reshaped matrix must have the same',\n",
       "  'number of entries.) Neither of these is standard mathematical notation, but they',\n",
       "  'can be useful in Julia.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '6.1 Matrices 49',\n",
       "  '',\n",
       "  'julia> B = [ 1 -3 ; 2 0 ; 1 -2]',\n",
       "  '',\n",
       "  '3×2 Array{Int64,2}:',\n",
       "  '1 -3',\n",
       "  '',\n",
       "  '2 0',\n",
       "  '',\n",
       "  '1 -2',\n",
       "  '',\n",
       "  'julia> B[:]',\n",
       "  '',\n",
       "  '6-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '-3',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  '-2',\n",
       "  '',\n",
       "  'julia> reshape(B,(2,3))',\n",
       "  '',\n",
       "  '2×3 Array{Int64,2}:',\n",
       "  '1 1 0',\n",
       "  '',\n",
       "  '2 -3 -2',\n",
       "  '',\n",
       "  'julia> reshape(B,(3,3))',\n",
       "  '',\n",
       "  'ERROR: DimensionMismatch(\"new dimensions (3, 3) must be consistent',\n",
       "  '',\n",
       "  'with array size 6\")',\n",
       "  '',\n",
       "  'Block matrices. Block matrices are constructed in Julia very much as in the',\n",
       "  'standard mathematical notation in VMLS. You use ; to stack matrices, and a',\n",
       "  'space to do (horizontal) concatenation. We apply this to the example on page 109',\n",
       "  'of VMLS.',\n",
       "  '',\n",
       "  'julia> B = [ 0 2 3 ]; # 1x3 matrix',\n",
       "  '',\n",
       "  'julia> C = [ -1 ]; # 1x1 matrix',\n",
       "  '',\n",
       "  'julia> D = [ 2 2 1 ; 1 3 5]; # 2x3 matrix',\n",
       "  '',\n",
       "  'julia> E = [4 ; 4 ]; # 2x1 matrix',\n",
       "  '',\n",
       "  'julia> # construct 3x4 block matrix',\n",
       "  '',\n",
       "  'julia> A = [B C ;',\n",
       "  '',\n",
       "  'D E]',\n",
       "  '',\n",
       "  '3×4 Array{Int64,2}:',\n",
       "  '0 2 3 -1',\n",
       "  '',\n",
       "  '2 2 1 4',\n",
       "  '',\n",
       "  '1 3 5 4',\n",
       "  '',\n",
       "  'Column and row interpretation of a matrix. An m × n matrix A can be inter-',\n",
       "  'preted as a collection of n m-vectors (its columns) or a collection of m row vectors',\n",
       "  '(its rows). Julia distinguishes between a matrix (a two-dimensional array) and an',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#equation.6.1.1',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '50 6 Matrices',\n",
       "  '',\n",
       "  'array of vectors. An array (or a tuple) of column vectors can be converted into a',\n",
       "  'matrix using the horizontal concatenation function hcat.',\n",
       "  '',\n",
       "  'julia> a = [ [1., 2.], [4., 5.], [7., 8.] ] # array of 2-vectors',\n",
       "  '',\n",
       "  '3-element Array{Array{Float64,1},1}:',\n",
       "  '',\n",
       "  '[1.0, 2.0]',\n",
       "  '',\n",
       "  '[4.0, 5.0]',\n",
       "  '',\n",
       "  '[7.0, 8.0]',\n",
       "  '',\n",
       "  'julia> A = hcat(a...)',\n",
       "  '',\n",
       "  '2×3 Array{Float64,2}:',\n",
       "  '1.0 4.0 7.0',\n",
       "  '',\n",
       "  '2.0 5.0 8.0',\n",
       "  '',\n",
       "  'The ... operator in hcat(a...) splits the array a into its elements, i.e., hcat(a...)',\n",
       "  'is the same as hcat(a[1], a[2], a[3]), which concatenates a[1], a[2], a[3]',\n",
       "  'horizontally.',\n",
       "  '',\n",
       "  'Similarly, vcat concatenates an array of arrays vertically. This is useful when',\n",
       "  'constructing a matrix from its row vectors.',\n",
       "  '',\n",
       "  'julia> a = [ [1. 2.], [4. 5.], [7. 8.] ] # array of 1x2 matrices',\n",
       "  '',\n",
       "  '3-element Array{Array{Float64,2},1}:',\n",
       "  '',\n",
       "  '[1.0 2.0]',\n",
       "  '',\n",
       "  '[4.0 5.0]',\n",
       "  '',\n",
       "  '[7.0 8.0]',\n",
       "  '',\n",
       "  'julia> A = vcat(a...)',\n",
       "  '',\n",
       "  '3×2 Array{Float64,2}:',\n",
       "  '1.0 2.0',\n",
       "  '',\n",
       "  '4.0 5.0',\n",
       "  '',\n",
       "  '7.0 8.0',\n",
       "  '',\n",
       "  '6.2 Zero and identity matrices',\n",
       "  '',\n",
       "  'Zero matrices. A zero matrix of size m× n is created using zeros(m,n).',\n",
       "  '',\n",
       "  'julia> zeros(2,2)',\n",
       "  '',\n",
       "  '2×2 Array{Float64,2}:',\n",
       "  '0.0 0.0',\n",
       "  '',\n",
       "  '0.0 0.0',\n",
       "  '',\n",
       "  'Identity matrices. Identity matrices in Julia can be created many ways, for ex-',\n",
       "  'ample by starting with a zero matrix and then setting the diagonal entries to one.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '6.2 Zero and identity matrices 51',\n",
       "  '',\n",
       "  'The LinearAlgebra package also contains functions for creating a special identity',\n",
       "  'matrix object I, which has some nice features. You can use 1.0*Matrix(I,n,n) to',\n",
       "  'create an n× n identity matrix. (Multiplying by 1.0 converts the matrix into one',\n",
       "  'with numerical entries; otherwise it has Boolean entries.) This expression is pretty',\n",
       "  'unwieldy, so we can define a function eye(n) to generate an identity matrix. This',\n",
       "  'function is in the VMLS package, so you can use it once the package is installed. (The',\n",
       "  'name eye to denote the identity matrix I traces back to the MATLAB language.)',\n",
       "  '',\n",
       "  'julia> eye(n) = 1.0*Matrix(I,n,n)',\n",
       "  '',\n",
       "  'julia> eye(4)',\n",
       "  '',\n",
       "  '4×4 Array{Float64,2}:',\n",
       "  '1.0 0.0 0.0 0.0',\n",
       "  '',\n",
       "  '0.0 1.0 0.0 0.0',\n",
       "  '',\n",
       "  '0.0 0.0 1.0 0.0',\n",
       "  '',\n",
       "  '0.0 0.0 0.0 1.0',\n",
       "  '',\n",
       "  'Julia’s identity matrix I has some useful properties. For example, when it can',\n",
       "  'deduce its dimensions, you don’t have to specify it. (This is the same as with',\n",
       "  'common mathematical notation; see VMLS page 113.)',\n",
       "  '',\n",
       "  'julia> A = [ 1 -1 2; 0 3 -1]',\n",
       "  '',\n",
       "  '2×3 Array{Int64,2}:',\n",
       "  '1 -1 2',\n",
       "  '',\n",
       "  '0 3 -1',\n",
       "  '',\n",
       "  'julia> [A I]',\n",
       "  '',\n",
       "  '2×5 Array{Int64,2}:',\n",
       "  '1 -1 2 1 0',\n",
       "  '',\n",
       "  '0 3 -1 0 1',\n",
       "  '',\n",
       "  'julia> [A ; I]',\n",
       "  '',\n",
       "  '5×3 Array{Int64,2}:',\n",
       "  '1 -1 2',\n",
       "  '',\n",
       "  '0 3 -1',\n",
       "  '',\n",
       "  '1 0 0',\n",
       "  '',\n",
       "  '0 1 0',\n",
       "  '',\n",
       "  '0 0 1',\n",
       "  '',\n",
       "  'julia> B = [ 1 2 ; 3 4 ]',\n",
       "  '',\n",
       "  '2×2 Array{Int64,2}:',\n",
       "  '1 2',\n",
       "  '',\n",
       "  '3 4',\n",
       "  '',\n",
       "  'julia> B + I',\n",
       "  '',\n",
       "  '2×2 Array{Int64,2}:',\n",
       "  '2 2',\n",
       "  '',\n",
       "  '3 5',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.135',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '52 6 Matrices',\n",
       "  '',\n",
       "  'Ones matrix. In VMLS we do not have notation for a matrix with all entries one.',\n",
       "  'In Julia, such a matrix is given by ones(m,n).',\n",
       "  '',\n",
       "  'Diagonal matrices. In standard mathematical notation, diag(1, 2, 3) is a diagonal',\n",
       "  '3 × 3 matrix with diagonal entries 1, 2, 3. In Julia such a matrix is created using',\n",
       "  'the function diagm, provided in the LinearAlgebra package. To construct the',\n",
       "  'diagonal matrix with diagonal entries in the vector s, you use diagm(0 => s).',\n",
       "  'This is fairly unwieldy, so the VMLS package defines a function diagonal(s). (Note',\n",
       "  'that you have to pass the diagonal entries as a vector.)',\n",
       "  '',\n",
       "  'julia> diagonal(x) = diagm(0 => x)',\n",
       "  '',\n",
       "  'diagonal (generic function with 1 method)',\n",
       "  '',\n",
       "  'julia> diagonal([1,2,3])',\n",
       "  '',\n",
       "  '3×3 Array{Int64,2}:',\n",
       "  '1 0 0',\n",
       "  '',\n",
       "  '0 2 0',\n",
       "  '',\n",
       "  '0 0 3',\n",
       "  '',\n",
       "  'A closely related Julia function diag(X) does the opposite: It takes the diagonal',\n",
       "  'entries of the (possibly not square) matrix X and puts them into a vector.',\n",
       "  '',\n",
       "  'julia> H = [0 1 -2 1; 2 -1 3 0]',\n",
       "  '',\n",
       "  '2×4 Array{Int64,2}:',\n",
       "  '0 1 -2 1',\n",
       "  '',\n",
       "  '2 -1 3 0',\n",
       "  '',\n",
       "  'julia> diag(H)',\n",
       "  '',\n",
       "  '2-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  '-1',\n",
       "  '',\n",
       "  'Random matrices. A random m×n matrix with entries between 0 and 1 is created',\n",
       "  'using rand(m,n). For entries that have a normal distribution, randn(m,n).',\n",
       "  '',\n",
       "  'julia> rand(2,3)',\n",
       "  '',\n",
       "  '2×3 Array{Float64,2}:',\n",
       "  '0.365832 0.381598 0.321444',\n",
       "  '',\n",
       "  '0.0317522 0.434451 0.95419',\n",
       "  '',\n",
       "  'julia> randn(3,2)',\n",
       "  '',\n",
       "  '3×2 Array{Float64,2}:',\n",
       "  '0.541546 1.65458',\n",
       "  '',\n",
       "  '-0.684011 -2.12776',\n",
       "  '',\n",
       "  '0.0443909 -1.81297',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '6.2 Zero and identity matrices 53',\n",
       "  '',\n",
       "  'Sparse matrices. Functions for creating and manipulating sparse matrices are',\n",
       "  'contained in the SparseArrays package, which must be installed; see page ix.',\n",
       "  'Sparse matrices are stored in a special format that exploits the property that most',\n",
       "  'of the elements are zero. The sparse function creates a sparse matrix from three',\n",
       "  'arrays that specify the row indexes, column indexes, and values of the nonzero',\n",
       "  'elements. The following code creates a sparse matrix',\n",
       "  '',\n",
       "  'A =',\n",
       "  '',\n",
       "  '\\uf8ee\\uf8ef\\uf8ef\\uf8f0',\n",
       "  '−1.11 0 1.17 0 0',\n",
       "  '',\n",
       "  '0.15 −0.10 0 0 0',\n",
       "  '0 0 −0.30 0 0',\n",
       "  '0 0 0 0.13 0',\n",
       "  '',\n",
       "  '\\uf8f9\\uf8fa\\uf8fa\\uf8fb .',\n",
       "  '',\n",
       "  'julia> I = [ 1, 2, 2, 1, 3, 4 ] # row indexes of nonzeros',\n",
       "  '',\n",
       "  'julia> J = [ 1, 1, 2, 3, 3, 4 ] # column indexes',\n",
       "  '',\n",
       "  'julia> V = [ -1.11, 0.15, -0.10, 1.17, -0.30, 0.13 ] # values',\n",
       "  '',\n",
       "  'julia> A = sparse(I, J, V, 4, 5)',\n",
       "  '',\n",
       "  '4×5 SparseMatrixCSC{Float64,Int64} with 6 stored entries:',\n",
       "  '[1, 1] = -1.11',\n",
       "  '',\n",
       "  '[2, 1] = 0.15',\n",
       "  '',\n",
       "  '[2, 2] = -0.1',\n",
       "  '',\n",
       "  '[1, 3] = 1.17',\n",
       "  '',\n",
       "  '[3, 3] = -0.3',\n",
       "  '',\n",
       "  '[4, 4] = 0.13',\n",
       "  '',\n",
       "  'julia> nnz(A)',\n",
       "  '',\n",
       "  '6',\n",
       "  '',\n",
       "  'Sparse matrices can be converted to regular non-sparse matrices using the Array',\n",
       "  'function. Applying sparse to a full matrix gives the equivalent sparse matrix.',\n",
       "  '',\n",
       "  'julia> A = sparse([1, 3, 2, 1], [1, 1, 2, 3],',\n",
       "  '',\n",
       "  '[1.0, 2.0, 3.0, 4.0], 3, 3)',\n",
       "  '',\n",
       "  'julia> 3×3 SparseMatrixCSC{Float64,Int64} with 4 stored entries:',\n",
       "  '[1, 1] = 1.0',\n",
       "  '',\n",
       "  '[3, 1] = 2.0',\n",
       "  '',\n",
       "  '[2, 2] = 3.0',\n",
       "  '',\n",
       "  '[1, 3] = 4.0',\n",
       "  '',\n",
       "  'julia> B = Array(A)',\n",
       "  '',\n",
       "  '3×3 Array{Float64,2}:',\n",
       "  '1.0 0.0 4.0',\n",
       "  '',\n",
       "  '0.0 3.0 0.0',\n",
       "  '',\n",
       "  '2.0 0.0 0.0',\n",
       "  '',\n",
       "  'julia> B[1,3] = 0.0;',\n",
       "  '',\n",
       "  'julia> sparse(B)',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '54 6 Matrices',\n",
       "  '',\n",
       "  '3×3 SparseMatrixCSC{Float64,Int64} with 3 stored entries:',\n",
       "  '[1, 1] = 1.0',\n",
       "  '',\n",
       "  '[3, 1] = 2.0',\n",
       "  '',\n",
       "  '[2, 2] = 3.0',\n",
       "  '',\n",
       "  'A sparse m× n zero matrix is created with spzeros(m,n). To create a sparse',\n",
       "  'n × n identity matrix in Julia, use sparse(1.0I,n,n). This is not a particularly',\n",
       "  'natural syntax, so we define a function speye(n) in the VMLS package. The VMLS',\n",
       "  'package also includes the function speye(n) which creates a sparse n× n identity',\n",
       "  'matrix, as well as spdiagonal(a), which creates a sparse diagonal matrix with the',\n",
       "  'entries of the vector a on its diagonal.',\n",
       "  '',\n",
       "  'A useful function for creating a random sparse matrix is sprand(m,n,d) (with',\n",
       "  'entries between 0 and 1) and sprandn(m,n,d) (with entries that range over all',\n",
       "  'numbers). The first two arguments give the dimensions of the matrix; the last one,',\n",
       "  'd, gives the density of nonzero entries. The nonzero entries are chosen randomly,',\n",
       "  'with about mnd of them nonzero. The following code creates a random 10000 ×',\n",
       "  '10000 sparse matrix, with a density 10−7. This means that we’d expect there to',\n",
       "  'be around 10 nonzero entries. (So this is a very sparse matrix!)',\n",
       "  '',\n",
       "  'julia> A = sprand(10000,10000,10^-7)',\n",
       "  '',\n",
       "  '10000×10000 SparseMatrixCSC{Float64,Int64} with 10 stored entries:',\n",
       "  '[6435 , 513] = 0.912633',\n",
       "  '',\n",
       "  '[3274 , 1518] = 0.369523',\n",
       "  '',\n",
       "  '[8264 , 2169] = 0.875005',\n",
       "  '',\n",
       "  '[8029 , 3513] = 0.0670725',\n",
       "  '',\n",
       "  '[7427 , 4122] = 0.0376876',\n",
       "  '',\n",
       "  '[6329 , 5078] = 0.970446',\n",
       "  '',\n",
       "  '[3882 , 5213] = 0.0964994',\n",
       "  '',\n",
       "  '[102 , 6572] = 0.697033',\n",
       "  '',\n",
       "  '[730 , 7936] = 0.489414',\n",
       "  '',\n",
       "  '[7006 , 8422] = 0.909351',\n",
       "  '',\n",
       "  '6.3 Transpose, addition, and norm',\n",
       "  '',\n",
       "  'Transpose. In VMLS we denote the transpose of an m × n matrix A as AT . In',\n",
       "  \"Julia, the transpose of A is given by A'.\",\n",
       "  '',\n",
       "  'julia> H = [0 1 -2 1; 2 -1 3 0]',\n",
       "  '',\n",
       "  '2×4 Array{Int64,2}:',\n",
       "  '0 1 -2 1',\n",
       "  '',\n",
       "  '2 -1 3 0',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '6.3 Transpose, addition, and norm 55',\n",
       "  '',\n",
       "  \"julia> H'\",\n",
       "  '4×2 Array{Int64,2}:',\n",
       "  '0 2',\n",
       "  '',\n",
       "  '1 -1',\n",
       "  '',\n",
       "  '-2 3',\n",
       "  '',\n",
       "  '1 0',\n",
       "  '',\n",
       "  'Addition, subtraction, and scalar multiplication. In Julia, addition and subtrac-',\n",
       "  'tion of matrices, and scalar-matrix multiplication, both follow standard mathemat-',\n",
       "  'ical notation.',\n",
       "  '',\n",
       "  'julia> U = [ 0 4; 7 0; 3 1]',\n",
       "  '',\n",
       "  '3×2 Array{Int64,2}:',\n",
       "  '0 4',\n",
       "  '',\n",
       "  '7 0',\n",
       "  '',\n",
       "  '3 1',\n",
       "  '',\n",
       "  'julia> V = [ 1 2; 2 3; 0 4]',\n",
       "  '',\n",
       "  '3×2 Array{Int64,2}:',\n",
       "  '1 2',\n",
       "  '',\n",
       "  '2 3',\n",
       "  '',\n",
       "  '0 4',\n",
       "  '',\n",
       "  'julia> U+V',\n",
       "  '',\n",
       "  '3×2 Array{Int64,2}:',\n",
       "  '1 6',\n",
       "  '',\n",
       "  '9 3',\n",
       "  '',\n",
       "  '3 5',\n",
       "  '',\n",
       "  'julia> 2.2*U',\n",
       "  '',\n",
       "  '3×2 Array{Float64,2}:',\n",
       "  '0.0 8.8',\n",
       "  '',\n",
       "  '15.4 0.0',\n",
       "  '',\n",
       "  '6.6 2.2',\n",
       "  '',\n",
       "  '(We can also multiply a matrix on the right by a scalar.)',\n",
       "  'Julia supports some operations that are not standard mathematical ones. For',\n",
       "  '',\n",
       "  'example, in Julia you can add or subtract a constant from a matrix, which carries',\n",
       "  'out the operation on each entry.',\n",
       "  '',\n",
       "  'Elementwise operations. The syntax for elementwise vector operations described',\n",
       "  'on page 10 carries over naturally to matrices. We add a period before a binary',\n",
       "  'operator to change the interpretation to elementwise. For example, if A and B',\n",
       "  'are matrices of the same size, then C = A .* B creates a matrix of the same size',\n",
       "  'with elements Cij = AijBij . We can add a period after a function name to tell',\n",
       "  'Julia that the function should be applied elementwise. Thus, if X is a matrix, then',\n",
       "  'Y = exp.(X) creates a matrix of the same size, with elements Yij = exp(Xij).',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '56 6 Matrices',\n",
       "  '',\n",
       "  'Matrix norm. In VMLS we use ‖A‖ to denote the norm of an m× n matrix,',\n",
       "  '',\n",
       "  '‖A‖ =',\n",
       "  '',\n",
       "  '\\uf8eb\\uf8ed m∑',\n",
       "  'i=1',\n",
       "  '',\n",
       "  'n∑',\n",
       "  'j=1',\n",
       "  '',\n",
       "  'A2ij',\n",
       "  '',\n",
       "  '\\uf8f6\\uf8f81/2 .',\n",
       "  'In standard mathematical notation, this is more often written as ‖A‖F , where F',\n",
       "  'stands for the name Frobenius. In standard mathematical notation, ‖A‖ usually',\n",
       "  'refers to another norm of a matrix, that is beyond the scope of the topics in VMLS.',\n",
       "  'In Julia, norm(A) gives the norm used in VMLS.',\n",
       "  '',\n",
       "  'julia> A = [2 3 -1; 0 -1 4]',\n",
       "  '',\n",
       "  'julia> norm(A)',\n",
       "  '',\n",
       "  '5.5677643628300215',\n",
       "  '',\n",
       "  'julia> norm(A[:])',\n",
       "  '',\n",
       "  '5.5677643628300215',\n",
       "  '',\n",
       "  'Triangle inequality. Let’s check that the triangle inequality ‖A+B‖ ≤ ‖A‖+‖B‖',\n",
       "  'holds, for two specific matrices.',\n",
       "  '',\n",
       "  'julia> A = [-1 0; 2 2]; B= [3 1; -3 2];',\n",
       "  '',\n",
       "  'julia> norm(A + B), norm(A) + norm(B)',\n",
       "  '',\n",
       "  '(4.69041575982343, 7.795831523312719)',\n",
       "  '',\n",
       "  '6.4 Matrix-vector multiplication',\n",
       "  '',\n",
       "  'In Julia, matrix-vector multiplication has the natural syntax y=A*x.',\n",
       "  '',\n",
       "  'julia> A = [0 2 -1; -2 1 1]',\n",
       "  '',\n",
       "  '2×3 Array{Int64,2}:',\n",
       "  '0 2 -1',\n",
       "  '',\n",
       "  '-2 1 1',\n",
       "  '',\n",
       "  'julia> x = [2, 1, -1]',\n",
       "  '',\n",
       "  '3-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '-1',\n",
       "  '',\n",
       "  'julia> A*x',\n",
       "  '',\n",
       "  '2-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '3',\n",
       "  '',\n",
       "  '-4',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '6.4 Matrix-vector multiplication 57',\n",
       "  '',\n",
       "  'Difference matrix. An (n − 1) × n difference matrix (equation (6.5) of VMLS)',\n",
       "  'can be constructed in several ways. A simple one is the following.',\n",
       "  '',\n",
       "  'julia> difference_matrix(n) = [-eye(n-1) zeros(n-1)] +',\n",
       "  '',\n",
       "  '[zeros(n-1), eye(n-1)];',\n",
       "  '',\n",
       "  'julia> D = difference_matrix(4)',\n",
       "  '',\n",
       "  '3×4 Array{Float64,2}:',\n",
       "  '-1.0 1.0 0.0 0.0',\n",
       "  '',\n",
       "  '0.0 -1.0 1.0 0.0',\n",
       "  '',\n",
       "  '0.0 0.0 -1.0 1.0',\n",
       "  '',\n",
       "  'julia> D*[-1,0,2,1]',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '1.0',\n",
       "  '',\n",
       "  '2.0',\n",
       "  '',\n",
       "  '-1.0',\n",
       "  '',\n",
       "  'Since a difference matrix contains many zeros, this is a good opportunity to use',\n",
       "  'sparse matrices.',\n",
       "  '',\n",
       "  'julia> difference_matrix(n) = [-speye(n-1) spzeros(n-1)] +',\n",
       "  '',\n",
       "  '[spzeros(n-1) speye(n-1)];',\n",
       "  '',\n",
       "  'julia> D = difference_matrix(4)',\n",
       "  '',\n",
       "  '3×4 SparseMatrixCSC{Float64,Int64} with 6 stored entries:',\n",
       "  '[1, 1] = -1.0',\n",
       "  '',\n",
       "  '[1, 2] = 1.0',\n",
       "  '',\n",
       "  '[2, 2] = -1.0',\n",
       "  '',\n",
       "  '[2, 3] = 1.0',\n",
       "  '',\n",
       "  '[3, 3] = -1.0',\n",
       "  '',\n",
       "  '[3, 4] = 1.0',\n",
       "  '',\n",
       "  'julia> D*[-1,0,2,1]',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '1.0',\n",
       "  '',\n",
       "  '2.0',\n",
       "  '',\n",
       "  '-1.0',\n",
       "  '',\n",
       "  'Running sum matrix. The running sum matrix (equation (6.6) in VMLS) is a',\n",
       "  'lower triangular matrix, with elements on and below the diagonal equal to one.',\n",
       "  '',\n",
       "  'julia> function running_sum(n) # n x n running sum matrix',\n",
       "  '',\n",
       "  'S = zeros(n,n)',\n",
       "  '',\n",
       "  'for i=1:n',\n",
       "  '',\n",
       "  'for j=1:i',\n",
       "  '',\n",
       "  'S[i,j] = 1',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#equation.6.4.5',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#equation.6.4.6',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '58 6 Matrices',\n",
       "  '',\n",
       "  'end',\n",
       "  '',\n",
       "  'end',\n",
       "  '',\n",
       "  'return S',\n",
       "  '',\n",
       "  'end',\n",
       "  '',\n",
       "  'running_sum (generic function with 1 method)',\n",
       "  '',\n",
       "  'julia> running_sum(4)',\n",
       "  '',\n",
       "  '4×4 Array{Float64,2}:',\n",
       "  '1.0 0.0 0.0 0.0',\n",
       "  '',\n",
       "  '1.0 1.0 0.0 0.0',\n",
       "  '',\n",
       "  '1.0 1.0 1.0 0.0',\n",
       "  '',\n",
       "  '1.0 1.0 1.0 1.0',\n",
       "  '',\n",
       "  'julia> running_sum(4)*[-1,1,2,0]',\n",
       "  '',\n",
       "  '4-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '-1.0',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '2.0',\n",
       "  '',\n",
       "  '2.0',\n",
       "  '',\n",
       "  'An alternative construction is tril(ones(n,n)). This uses the function tril,',\n",
       "  'which sets the elements of a matrix above the diagonal to zero.',\n",
       "  '',\n",
       "  'Vandermonde matrix. An m×n Vandermonde matrix (equation (6.7) in VMLS)',\n",
       "  'has entries tj−1i for i = 1, . . . ,m and j = 1, . . . , n. We define a function that',\n",
       "  'takes an m-vector with elements t1, . . . , tm and returns the corresponding m × n',\n",
       "  'Vandermonde matrix.',\n",
       "  '',\n",
       "  'julia> function vandermonde(t,n)',\n",
       "  '',\n",
       "  'm = length(t)',\n",
       "  '',\n",
       "  'V = zeros(m,n)',\n",
       "  '',\n",
       "  'for i=1:m',\n",
       "  '',\n",
       "  'for j=1:n',\n",
       "  '',\n",
       "  'V[i,j] = t[i]^(j-1)',\n",
       "  '',\n",
       "  'end',\n",
       "  '',\n",
       "  'end',\n",
       "  '',\n",
       "  'return V',\n",
       "  '',\n",
       "  'end',\n",
       "  '',\n",
       "  'vandermonde (generic function with 1 method)',\n",
       "  '',\n",
       "  'julia> vandermonde([-1,0,0.5,1],5)',\n",
       "  '',\n",
       "  '4×5 Array{Float64,2}:',\n",
       "  '1.0 -1.0 1.0 -1.0 1.0',\n",
       "  '',\n",
       "  '1.0 0.0 0.0 0.0 0.0',\n",
       "  '',\n",
       "  '1.0 0.5 0.25 0.125 0.0625',\n",
       "  '',\n",
       "  '1.0 1.0 1.0 1.0 1.0',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#equation.6.4.7',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '6.5 Complexity 59',\n",
       "  '',\n",
       "  'An alternative shorter definition uses Julia’s hcat function.',\n",
       "  '',\n",
       "  'julia> vandermonde(t,n) = hcat( [t.^i for i = 0:n-1]... )',\n",
       "  '',\n",
       "  'vandermonde (generic function with 1 method)',\n",
       "  '',\n",
       "  'julia> vandermonde([-1,0,0.5,1],5)',\n",
       "  '',\n",
       "  '4×5 Array{Float64,2}:',\n",
       "  '1.0 -1.0 1.0 -1.0 1.0',\n",
       "  '',\n",
       "  '1.0 0.0 0.0 0.0 0.0',\n",
       "  '',\n",
       "  '1.0 0.5 0.25 0.125 0.0625',\n",
       "  '',\n",
       "  '1.0 1.0 1.0 1.0 1.0',\n",
       "  '',\n",
       "  '6.5 Complexity',\n",
       "  '',\n",
       "  'Complexity of matrix-vector multiplication. The complexity of multiplying an',\n",
       "  'm×n matrix by an n-vector is 2mn flops. This grows linearly with both m and n.',\n",
       "  'Let’s check this.',\n",
       "  '',\n",
       "  'julia> A = rand(1000,10000); x = rand(10000);',\n",
       "  '',\n",
       "  'julia> @time y = A*x;',\n",
       "  '',\n",
       "  '0.022960 seconds (2.01 k allocations: 127.499 KiB)',\n",
       "  '',\n",
       "  'julia> @time y = A*x;',\n",
       "  '',\n",
       "  '0.006321 seconds (5 allocations: 8.094 KiB)',\n",
       "  '',\n",
       "  'julia> A = rand(5000,20000); x = rand(20000);',\n",
       "  '',\n",
       "  'julia> @time y = A*x;',\n",
       "  '',\n",
       "  '0.084710 seconds (6 allocations: 39.297 KiB)',\n",
       "  '',\n",
       "  'julia> @time y = A*x;',\n",
       "  '',\n",
       "  '0.047996 seconds (6 allocations: 39.297 KiB)',\n",
       "  '',\n",
       "  'In the second matrix-vector multiply, m increases by a factor of 5 and n increases',\n",
       "  'by a factor of 2, so the complexity predicts that the computation time should be',\n",
       "  '(approximately) increased by a factor of 10. As we can see, it is increased by a',\n",
       "  'factor around 7.4.',\n",
       "  '',\n",
       "  'The increase in efficiency obtained by sparse matrix computations is seen from',\n",
       "  'matrix-vector multiplications with the difference matrix.',\n",
       "  '',\n",
       "  'julia> n = 10^4;',\n",
       "  '',\n",
       "  'julia> D = [-eye(n-1) zeros(n-1)] + [zeros(n-1) eye(n-1)];',\n",
       "  '',\n",
       "  'julia> x = randn(n);',\n",
       "  '',\n",
       "  'julia> @time y=D*x;',\n",
       "  '',\n",
       "  '0.051516 seconds (6 allocations: 78.359 KiB)',\n",
       "  '',\n",
       "  'julia> Ds = [-speye(n-1) spzeros(n-1)] + [spzeros(n-1) speye(n-1)];',\n",
       "  '',\n",
       "  'julia> @time y=Ds*x;',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '60 6 Matrices',\n",
       "  '',\n",
       "  '0.000177 seconds (6 allocations: 78.359 KiB)',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Chapter 7',\n",
       "  '',\n",
       "  'Matrix examples',\n",
       "  '',\n",
       "  '7.1 Geometric transformations',\n",
       "  '',\n",
       "  'Let’s create a rotation matrix, and use it to rotate a set of points π/3 radians (60◦).',\n",
       "  'The result is in Figure 7.1.',\n",
       "  '',\n",
       "  'julia> Rot(theta) = [cos(theta) -sin(theta); sin(theta) cos(theta)];',\n",
       "  '',\n",
       "  'julia> R = Rot(pi/3)',\n",
       "  '',\n",
       "  '2×2 Array{Float64,2}:',\n",
       "  '0.5 -0.866025',\n",
       "  '',\n",
       "  '0.866025 0.5',\n",
       "  '',\n",
       "  'julia> # Create a list of 2-D points',\n",
       "  '',\n",
       "  'julia> points = [ [1,0], [1.5,0], [2,0], [1,0.25], [1.5, 0.25],',\n",
       "  '',\n",
       "  '[1,.5] ];',\n",
       "  '',\n",
       "  'julia> # Now rotate them.',\n",
       "  '',\n",
       "  'julia> rpoints = [ R*p for p in points ];',\n",
       "  '',\n",
       "  'julia> # Show the two sets of points.',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> scatter([c[1] for c in points], [c[2] for c in points])',\n",
       "  '',\n",
       "  'julia> scatter!([c[1] for c in rpoints], [c[2] for c in rpoints])',\n",
       "  '',\n",
       "  'julia> plot!(lims = (-0.1, 2.1), size = (500,500), legend = false)',\n",
       "  '',\n",
       "  '7.2 Selectors',\n",
       "  '',\n",
       "  'Reverser matrix. The reverser matrix can be created from an identity matrix',\n",
       "  'by reversing the order of its rows. The Julia command reverse can be used for',\n",
       "  'this purpose. (reverse(A,dims=1) reverses the order of the rows of a matrix;',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '62 7 Matrix examples',\n",
       "  '',\n",
       "  '0.0 0.5 1.0 1.5 2.0',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '0.5',\n",
       "  '',\n",
       "  '1.0',\n",
       "  '',\n",
       "  '1.5',\n",
       "  '',\n",
       "  '2.0',\n",
       "  '',\n",
       "  'Figure 7.1 Counterclockwise rotation by 60 degrees applied to six points.',\n",
       "  '',\n",
       "  'flipdim(A,dims=2) reverses the order of the columns.) Multiplying a vector with',\n",
       "  'a reverser matrix is the same as reversing the order of its entries directly.',\n",
       "  '',\n",
       "  'julia> reverser(n) = reverse(eye(n),dims=1)',\n",
       "  '',\n",
       "  'reverser (generic function with 1 method)',\n",
       "  '',\n",
       "  'julia> A = reverser(5)',\n",
       "  '',\n",
       "  '5×5 Array{Float64,2}:',\n",
       "  '0.0 0.0 0.0 0.0 1.0',\n",
       "  '',\n",
       "  '0.0 0.0 0.0 1.0 0.0',\n",
       "  '',\n",
       "  '0.0 0.0 1.0 0.0 0.0',\n",
       "  '',\n",
       "  '0.0 1.0 0.0 0.0 0.0',\n",
       "  '',\n",
       "  '1.0 0.0 0.0 0.0 0.0',\n",
       "  '',\n",
       "  'julia> x = [1., 2., 3., 4., 5.];',\n",
       "  '',\n",
       "  'julia> A*x # Reverse x by multiplying with reverser matrix.',\n",
       "  '',\n",
       "  '5-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '5.0',\n",
       "  '',\n",
       "  '4.0',\n",
       "  '',\n",
       "  '3.0',\n",
       "  '',\n",
       "  '2.0',\n",
       "  '',\n",
       "  '1.0',\n",
       "  '',\n",
       "  'julia> reverse(x) # Reverse x directly.',\n",
       "  '',\n",
       "  '5-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '5.0',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '7.3 Incidence matrix 63',\n",
       "  '',\n",
       "  '4.0',\n",
       "  '',\n",
       "  '3.0',\n",
       "  '',\n",
       "  '2.0',\n",
       "  '',\n",
       "  '1.0',\n",
       "  '',\n",
       "  'Permutation matrix. Let’s create a permutation matrix and use it to permute',\n",
       "  'the entries of a vector. In Julia, there is no reason to create a matrix to carry',\n",
       "  'out the permutation, since we can do the same thing directly by passing in the',\n",
       "  'permuted indexes to the vector.',\n",
       "  '',\n",
       "  'julia> A = [0 0 1; 1 0 0; 0 1 0]',\n",
       "  '',\n",
       "  '3×3 Array{Int64,2}:',\n",
       "  '0 0 1',\n",
       "  '',\n",
       "  '1 0 0',\n",
       "  '',\n",
       "  '0 1 0',\n",
       "  '',\n",
       "  'julia> x = [0.2, -1.7, 2.4]',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.2',\n",
       "  '',\n",
       "  '-1.7',\n",
       "  '',\n",
       "  '2.4',\n",
       "  '',\n",
       "  'julia> A*x # Permutes entries of x to [x[3],x[1],x[2]]',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '2.4',\n",
       "  '',\n",
       "  '0.2',\n",
       "  '',\n",
       "  '-1.7',\n",
       "  '',\n",
       "  'julia> x[[3,1,2]] # Same thing using permuted indices',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '2.4',\n",
       "  '',\n",
       "  '0.2',\n",
       "  '',\n",
       "  '-1.7',\n",
       "  '',\n",
       "  '7.3 Incidence matrix',\n",
       "  '',\n",
       "  'Incidence matrix of a graph. We create the incidence matrix of the network',\n",
       "  'shown in Figure 7.3 in VMLS.',\n",
       "  '',\n",
       "  'julia> A = [ -1 -1 0 1 0; 1 0 -1 0 0 ; 0 0 1 -1 -1 ; 0 1 0 0 1]',\n",
       "  '',\n",
       "  '4×5 Array{Int64,2}:',\n",
       "  '-1 -1 0 1 0',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.7.3',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '64 7 Matrix examples',\n",
       "  '',\n",
       "  '1 0 -1 0 0',\n",
       "  '',\n",
       "  '0 0 1 -1 -1',\n",
       "  '',\n",
       "  '0 1 0 0 1',\n",
       "  '',\n",
       "  'julia> xcirc = [1, -1, 1, 0, 1] # A circulation',\n",
       "  '',\n",
       "  '5-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '-1',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  'julia> A*xcirc',\n",
       "  '',\n",
       "  '4-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  'julia> s = [1,0,-1,0]; # A source vector',\n",
       "  '',\n",
       "  'julia> x = [0.6, 0.3, 0.6, -0.1, -0.3]; # A flow vector',\n",
       "  '',\n",
       "  'julia> A*x + s # Total incoming flow at each node',\n",
       "  '',\n",
       "  '4-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '1.11022e-16',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  'Dirichlet energy. On page 135 of VMLS we compute the Dirichlet energy of two',\n",
       "  'potential vectors associated with the graph of Figure 7.2 in VMLS.',\n",
       "  '',\n",
       "  'julia> A = [ -1 -1 0 1 0 ; 1 0 -1 0 0 ; 0 0 1 -1 -1; 0 1 0 0 1 ]',\n",
       "  '',\n",
       "  '4×5 Array{Int64,2}:',\n",
       "  '-1 -1 0 1 0',\n",
       "  '',\n",
       "  '1 0 -1 0 0',\n",
       "  '',\n",
       "  '0 0 1 -1 -1',\n",
       "  '',\n",
       "  '0 1 0 0 1',\n",
       "  '',\n",
       "  'julia> vsmooth = [ 1, 2, 2, 1 ]',\n",
       "  '',\n",
       "  \"julia> norm(A'*vsmooth)^2 # Dirichlet energy of vsmooth\",\n",
       "  '2.9999999999999996',\n",
       "  '',\n",
       "  'julia> vrough = [ 1, -1, 2, -1 ]',\n",
       "  '',\n",
       "  \"julia> norm(A'*vrough)^2 # Dirichlet energy of vrough\",\n",
       "  '27.0',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.172',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.7.2',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '7.4 Convolution 65',\n",
       "  '',\n",
       "  '7.4 Convolution',\n",
       "  '',\n",
       "  'The Julia package DSP includes a convolution function conv. After adding this',\n",
       "  'package, the command conv(a,b) can be used to compute the convolution of the',\n",
       "  'vectors a and b. Let’s use this to find the coefficients of the polynomial',\n",
       "  '',\n",
       "  'p(x) = (1 + x)(2− x+ x2)(1 + x− 2x2) = 2 + 3x− 3x2 − x3 + x4 − 2x5.',\n",
       "  '',\n",
       "  'julia> Using DSP',\n",
       "  '',\n",
       "  'julia> a = [1,1]; # coefficients of 1+x',\n",
       "  '',\n",
       "  'julia> b = [2,-1,1]; # coefficients of 2-x+x^2',\n",
       "  '',\n",
       "  'julia> c = [1,1,-2]; # coefficients of 1+x-2x^2',\n",
       "  '',\n",
       "  'julia> d = conv(conv(a,b),c) # coefficients of product',\n",
       "  '',\n",
       "  '6-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  '3',\n",
       "  '',\n",
       "  '-3',\n",
       "  '',\n",
       "  '-1',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '-2',\n",
       "  '',\n",
       "  'Let’s write a function that creates a Toeplitz matrix, and check it against the',\n",
       "  'conv function. We will also check that Julia is using the very efficient method for',\n",
       "  'computing the convolution.',\n",
       "  '',\n",
       "  'To construct the Toeplitz matrix T (b) defined in equation (7.3) of VMLS, we',\n",
       "  'first create a zero matrix of the correct dimensions ((n + m − 1) × n) and then',\n",
       "  'add the coefficients bi one by one. Single-index indexing comes in handy for this',\n",
       "  'purpose. The single-index indexes of the elements bi in the matrix T (b) are i,',\n",
       "  'i+m+ n, i+ 2(m+ n), . . . , i+ (n− 1)(m+ n).',\n",
       "  '',\n",
       "  'julia> function toeplitz(b,n)',\n",
       "  '',\n",
       "  'm = length(b)',\n",
       "  '',\n",
       "  'T = zeros(n+m-1,n)',\n",
       "  '',\n",
       "  'for i=1:m',\n",
       "  '',\n",
       "  'T[i : n+m : end] .= b[i]',\n",
       "  '',\n",
       "  'end',\n",
       "  '',\n",
       "  'return T',\n",
       "  '',\n",
       "  'end',\n",
       "  '',\n",
       "  'julia> b = [-1,2,3]; a = [-2,3,-1,1];',\n",
       "  '',\n",
       "  'julia> Tb = toeplitz(b, length(a))',\n",
       "  '',\n",
       "  '6×4 Array{Float64,2}:',\n",
       "  '-1.0 0.0 0.0 0.0',\n",
       "  '',\n",
       "  '2.0 -1.0 0.0 0.0',\n",
       "  '',\n",
       "  '3.0 2.0 -1.0 0.0',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#equation.7.4.3',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '66 7 Matrix examples',\n",
       "  '',\n",
       "  '0.0 3.0 2.0 -1.0',\n",
       "  '',\n",
       "  '0.0 0.0 3.0 2.0',\n",
       "  '',\n",
       "  '0.0 0.0 0.0 3.0',\n",
       "  '',\n",
       "  'julia> Tb*a, conv(b,a)',\n",
       "  '',\n",
       "  '([2.0, -7.0, 1.0, 6.0, -1.0, 3.0], [2, -7, 1, 6, -1, 3])',\n",
       "  '',\n",
       "  'julia> m = 2000; n = 2000;',\n",
       "  '',\n",
       "  'julia> b = randn(n); a=randn(m);',\n",
       "  '',\n",
       "  'julia> @time ctoep = toeplitz(b,n)*a;',\n",
       "  '',\n",
       "  '0.124865 seconds (8.01 k allocations: 122.696 MiB, 5.07% gc time)',\n",
       "  '',\n",
       "  'julia> @time cconv = conv(a,b);',\n",
       "  '',\n",
       "  '0.000748 seconds (164 allocations: 259.313 KiB)',\n",
       "  '',\n",
       "  'julia> norm(ctoep - cconv)',\n",
       "  '',\n",
       "  '2.4593600404835336e-12',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Chapter 8',\n",
       "  '',\n",
       "  'Linear equations',\n",
       "  '',\n",
       "  '8.1 Linear and affine functions',\n",
       "  '',\n",
       "  'Matrix-vector product function. Let’s define an instance of the matrix-vector',\n",
       "  'product function, and then numerically check that superpoisition holds.',\n",
       "  '',\n",
       "  'julia> A = [-0.1 2.8 -1.6; 2.3 -0.6 -3.6] # Define 2x3 matrix A',\n",
       "  '',\n",
       "  '2×3 Array{Float64,2}:',\n",
       "  '-0.1 2.8 -1.6',\n",
       "  '',\n",
       "  '2.3 -0.6 -3.6',\n",
       "  '',\n",
       "  'julia> f(x) = A*x # Define matrix-vector product function',\n",
       "  '',\n",
       "  'f (generic function with 1 method)',\n",
       "  '',\n",
       "  'julia> # Let’s check superposition',\n",
       "  '',\n",
       "  'julia> x = [1, 2, 3]; y = [-3, -1, 2];',\n",
       "  '',\n",
       "  'julia> alpha = 0.5; beta = -1.6;',\n",
       "  '',\n",
       "  'julia> lhs = f(alpha*x+beta*y)',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '9.47',\n",
       "  '',\n",
       "  '16.75',\n",
       "  '',\n",
       "  'julia> rhs = alpha*f(x)+beta*f(y)',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '9.47',\n",
       "  '',\n",
       "  '16.75',\n",
       "  '',\n",
       "  'julia> norm(lhs-rhs)',\n",
       "  '',\n",
       "  '1.7763568394002505e-15',\n",
       "  '',\n",
       "  'julia> f([0,1,0]) # Should be second column of A',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '2.8',\n",
       "  '',\n",
       "  '-0.6',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '68 8 Linear equations',\n",
       "  '',\n",
       "  'De-meaning matrix. Let’s create a de-meaning matrix, and check that it works',\n",
       "  'on a vector.',\n",
       "  '',\n",
       "  'julia> de_mean(n) = eye(n) .- 1/n; # De-meaning matrix',\n",
       "  '',\n",
       "  'julia> x = [0.2, 2.3, 1.0];',\n",
       "  '',\n",
       "  'julia> de_mean(length(x))*x # De-mean using matrix multiplication',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '-0.966667',\n",
       "  '',\n",
       "  '1.13333',\n",
       "  '',\n",
       "  '-0.166667',\n",
       "  '',\n",
       "  'julia> x .- avg(x) # De-mean by subtracting mean',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '-0.966667',\n",
       "  '',\n",
       "  '1.13333',\n",
       "  '',\n",
       "  '-0.166667',\n",
       "  '',\n",
       "  'Examples of functions that are not linear. The componentwise absolute value',\n",
       "  'and the sort function are examples of nonlinear functions. These functions are',\n",
       "  'easily computed by abs and sort. By default, the sort function sorts in increasing',\n",
       "  'order, but this can be changed by adding an optional keyword argument.',\n",
       "  '',\n",
       "  'julia> f(x) = abs.(x) # componentwise absolute value',\n",
       "  '',\n",
       "  'f (generic function with 1 method)',\n",
       "  '',\n",
       "  'julia> x = [1, 0]; y = [0, 1]; alpha = -1; beta = 2;',\n",
       "  '',\n",
       "  'julia> f(alpha*x + beta*y)',\n",
       "  '',\n",
       "  '2-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  'julia> alpha*f(x) + beta*f(y)',\n",
       "  '',\n",
       "  '2-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '-1',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  'julia> f(x) = sort(x, rev = true) # sort in decreasing order',\n",
       "  '',\n",
       "  'f (generic function with 1 method)',\n",
       "  '',\n",
       "  'julia> f(alpha*x + beta*y)',\n",
       "  '',\n",
       "  '2-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  '-1',\n",
       "  '',\n",
       "  'julia> alpha*f(x) + beta*f(y)',\n",
       "  '',\n",
       "  '2-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '8.2 Linear function models 69',\n",
       "  '',\n",
       "  '8.2 Linear function models',\n",
       "  '',\n",
       "  'Price elasticity of demand. Let’s use a price elasticity of demand matrix to pre-',\n",
       "  'dict the demand for three products when the prices are changed a bit. Using this',\n",
       "  'we can predict the change in total profit, given the manufacturing costs.',\n",
       "  '',\n",
       "  'julia> p = [10, 20, 15]; # Current prices',\n",
       "  '',\n",
       "  'julia> d = [5.6, 1.5, 8.6]; # Current demand (say in thousands)',\n",
       "  '',\n",
       "  'julia> c = [6.5, 11.2, 9.8]; # Cost to manufacture',\n",
       "  '',\n",
       "  \"julia> profit = (p-c)'*d # Current total profit\",\n",
       "  '77.51999999999998',\n",
       "  '',\n",
       "  'julia> # Demand elasticity matrix',\n",
       "  '',\n",
       "  'julia> E = [-0.3 0.1 -0.1; 0.1 -0.5 0.05 ; -0.1 0.05 -0.4]',\n",
       "  '',\n",
       "  '3×3 Array{Float64,2}:',\n",
       "  '-0.3 0.1 -0.1',\n",
       "  '',\n",
       "  '0.1 -0.5 0.05',\n",
       "  '',\n",
       "  '-0.1 0.05 -0.4',\n",
       "  '',\n",
       "  'julia> p_new = [9, 21, 14]; # Proposed new prices',\n",
       "  '',\n",
       "  'julia> delta_p = (p_new-p)./p # Fractional change in prices',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '-0.1',\n",
       "  '',\n",
       "  '0.05',\n",
       "  '',\n",
       "  '-0.0666667',\n",
       "  '',\n",
       "  'julia> delta_d = E*delta_p # Predicted fractional change in demand',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.0416667',\n",
       "  '',\n",
       "  '-0.0383333',\n",
       "  '',\n",
       "  '0.0391667',\n",
       "  '',\n",
       "  'julia> d_new = d .* (1 .+ delta_d) # Predicted new demand',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '5.833333333333333',\n",
       "  '',\n",
       "  '1.4425',\n",
       "  '',\n",
       "  '8.936833333333333',\n",
       "  '',\n",
       "  \"julia> profit_new = (p_new-c)'*d_new # Predicted new profit\",\n",
       "  '66.25453333333333',\n",
       "  '',\n",
       "  'If we trust the linear demand elasticity model, we should not make these price',\n",
       "  'changes.',\n",
       "  '',\n",
       "  'Taylor approximation. Consider the nonlinear function f : R2 → R2 given by',\n",
       "  '',\n",
       "  'f(x) =',\n",
       "  '',\n",
       "  '[',\n",
       "  '‖x− a‖',\n",
       "  '‖x− b‖',\n",
       "  '',\n",
       "  ']',\n",
       "  '=',\n",
       "  '',\n",
       "  '[ √',\n",
       "  '(x1 − a1)2 + (x2 − a2)2√',\n",
       "  '(x1 − b1)2 + (x2 − b2)2',\n",
       "  '',\n",
       "  ']',\n",
       "  '.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '70 8 Linear equations',\n",
       "  '',\n",
       "  'The two components of f give the distance of x to the points a and b. The function',\n",
       "  'is differentiable, except when x = a or x = b. Its derivative or Jacobian matrix is',\n",
       "  'given by',\n",
       "  '',\n",
       "  'Df(z) =',\n",
       "  '',\n",
       "  '\\uf8ee\\uf8ef\\uf8ef\\uf8f0',\n",
       "  '∂f1',\n",
       "  '∂x1',\n",
       "  '',\n",
       "  '(z)',\n",
       "  '∂f1',\n",
       "  '∂x2',\n",
       "  '',\n",
       "  '(z)',\n",
       "  '',\n",
       "  '∂f2',\n",
       "  '∂x1',\n",
       "  '',\n",
       "  '(z)',\n",
       "  '∂f2',\n",
       "  '∂x2',\n",
       "  '',\n",
       "  '(z)',\n",
       "  '',\n",
       "  '\\uf8f9\\uf8fa\\uf8fa\\uf8fb =',\n",
       "  '\\uf8ee\\uf8ef\\uf8ef\\uf8f0',\n",
       "  '',\n",
       "  'z1 − a1',\n",
       "  '‖z − a‖',\n",
       "  '',\n",
       "  'z2 − a2',\n",
       "  '‖z − a‖',\n",
       "  '',\n",
       "  'z1 − b1',\n",
       "  '‖z − b‖',\n",
       "  '',\n",
       "  'z2 − b2',\n",
       "  '‖z − b‖',\n",
       "  '',\n",
       "  '\\uf8f9\\uf8fa\\uf8fa\\uf8fb .',\n",
       "  'Let’s form the Taylor approximation of f for some specific values of a, b, and z,',\n",
       "  'and then check it against the true value of f at a few points near z.',\n",
       "  '',\n",
       "  'julia> f(x) = [ norm(x-a), norm(x-b) ];',\n",
       "  '',\n",
       "  \"julia> Df(z) = [ (z-a)' / norm(z-a) ; (z-b)' / norm(z-b) ];\",\n",
       "  'julia> f_hat(x) = f(z) + Df(z)*(x-z);',\n",
       "  '',\n",
       "  'julia> a = [1, 0]; b = [1, 1]; z = [0, 0];',\n",
       "  '',\n",
       "  'julia> f([0.1, 0.1])',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.905539',\n",
       "  '',\n",
       "  '1.27279',\n",
       "  '',\n",
       "  'julia> f_hat([0.1, 0.1])',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.9',\n",
       "  '',\n",
       "  '1.27279',\n",
       "  '',\n",
       "  'julia> f([0.5, 0.5])',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.707107',\n",
       "  '',\n",
       "  '0.707107',\n",
       "  '',\n",
       "  'julia> f_hat([0.5, 0.5])',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.5',\n",
       "  '',\n",
       "  '0.707107',\n",
       "  '',\n",
       "  'Regression model. We revisit the regression model for the house sales data in',\n",
       "  'Section 2.3. The model is',\n",
       "  '',\n",
       "  'ŷ = xTβ + v = β1x1 + β2x2 + v,',\n",
       "  '',\n",
       "  'where ŷ is the predicted house sale price, x1 is the house area in 1000 square feet,',\n",
       "  'and x2 is the number of bedrooms.',\n",
       "  '',\n",
       "  'In the following code we construct the 2 × 774 data matrix X and vector of',\n",
       "  'outcomes yd, for the N = 774 examples in the data set. We then calculate the',\n",
       "  'regression model predictions ŷd, the prediction errors rd, and the RMS prediction',\n",
       "  'error.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '8.3 Systems of linear equations 71',\n",
       "  '',\n",
       "  'julia> # parameters in regression model',\n",
       "  '',\n",
       "  'julia> beta = [148.73, -18.85]; v = 54.40;',\n",
       "  '',\n",
       "  'julia> D = house_sales_data();',\n",
       "  '',\n",
       "  'julia> yd = D[\"price\"]; # vector of outcomes',\n",
       "  '',\n",
       "  'julia> N = length(yd)',\n",
       "  '',\n",
       "  '774',\n",
       "  '',\n",
       "  'julia> X = [ D[\"area\"] D[\"beds\"] ]\\';',\n",
       "  'julia> size(X)',\n",
       "  '',\n",
       "  '(2, 774)',\n",
       "  '',\n",
       "  \"julia> ydhat = X'*beta .+ v; # vector of predicted outcomes\",\n",
       "  'julia> rd = yd - ydhat; # vector of predicted errors',\n",
       "  '',\n",
       "  'julia> rms(rd) # RMS prediction error',\n",
       "  '',\n",
       "  '74.84571862623022',\n",
       "  '',\n",
       "  'julia> # Compare with standard deviation of prices',\n",
       "  '',\n",
       "  'julia> stdev(yd)',\n",
       "  '',\n",
       "  '112.7821615975651',\n",
       "  '',\n",
       "  '8.3 Systems of linear equations',\n",
       "  '',\n",
       "  'Balancing chemical reactions. We verify the linear balancing equations on page 155',\n",
       "  'of VMLS, for the simple example of electrolysis of water.',\n",
       "  '',\n",
       "  'julia> R = [2 ; 1]',\n",
       "  '',\n",
       "  '2-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  'julia> P = [2 0 ; 0 2]',\n",
       "  '',\n",
       "  '2×2 Array{Int64,2}:',\n",
       "  '2 0',\n",
       "  '',\n",
       "  '0 2',\n",
       "  '',\n",
       "  'julia> # Check balancing coefficients [2,2,1]',\n",
       "  '',\n",
       "  'julia> coeff = [2,2,1];',\n",
       "  '',\n",
       "  'julia> [R -P]*coeff',\n",
       "  '',\n",
       "  '2-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.192',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '72 8 Linear equations',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Chapter 9',\n",
       "  '',\n",
       "  'Linear dynamical systems',\n",
       "  '',\n",
       "  '9.1 Linear dynamical systems',\n",
       "  '',\n",
       "  'Let’s simulate a time-invariant linear dynamic system',\n",
       "  '',\n",
       "  'xt+1 = Axt, t = 1, . . . , T,',\n",
       "  '',\n",
       "  'with dynamics matrix',\n",
       "  '',\n",
       "  'A =',\n",
       "  '',\n",
       "  '\\uf8ee\\uf8f0 0.97 0.10 −0.05−0.30 0.99 0.05',\n",
       "  '0.01 −0.04 0.96',\n",
       "  '',\n",
       "  '\\uf8f9\\uf8fb',\n",
       "  'and initial state x1 = (1, 0,−1). We store the state trajectory in the n× T matrix',\n",
       "  'state_traj, with the ith column xt. We plot the result in Figure 9.1.',\n",
       "  '',\n",
       "  'julia> x_1 = [1,0,-1]; # initial state',\n",
       "  '',\n",
       "  'julia> n = length(x_1); T = 50;',\n",
       "  '',\n",
       "  'julia> A = [ 0.97 0.10 -0.05 ; -0.3 0.99 0.05 ; 0.01 -0.04 0.96 ]',\n",
       "  '',\n",
       "  '3×3 Array{Float64,2}:',\n",
       "  '0.97 0.1 -0.05',\n",
       "  '',\n",
       "  '-0.3 0.99 0.05',\n",
       "  '',\n",
       "  '0.01 -0.04 0.96',\n",
       "  '',\n",
       "  'julia> state_traj = [x_1 zeros(n,T-1) ];',\n",
       "  '',\n",
       "  'julia> for t=1:T-1 # Dynamics recursion',\n",
       "  '',\n",
       "  'state_traj[:,t+1] = A*state_traj[:,t];',\n",
       "  '',\n",
       "  'end',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> plot(1:T, state_traj\\', xlabel = \"t\",',\n",
       "  'label = [\"(x_t)_1\", \"(x_t)_2\", \"(x_t)_3\"])',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '74 9 Linear dynamical systems',\n",
       "  '',\n",
       "  '0 10 20 30 40 50',\n",
       "  '',\n",
       "  '-2',\n",
       "  '',\n",
       "  '-1',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  't',\n",
       "  '',\n",
       "  '(x_t)_1',\n",
       "  '(x_t)_2',\n",
       "  '(x_t)_3',\n",
       "  '',\n",
       "  'Figure 9.1 Linear dynamical system simulation.',\n",
       "  '',\n",
       "  '9.2 Population dynamics',\n",
       "  '',\n",
       "  'We can create a population dynamics matrix with just one simple line of Ju-',\n",
       "  'lia. The following code predicts the 2020 population distribution in the US using',\n",
       "  'the data of Section 9.2 of VMLS, which are available through the VMLS function',\n",
       "  'population_data. The result is shown in Figure 9.2.',\n",
       "  '',\n",
       "  'julia> # Import 3 100-vectors: population, birth_rate, death_rate',\n",
       "  '',\n",
       "  'julia> D = population_data();',\n",
       "  '',\n",
       "  'julia> b = D[\"birth_rate\"];',\n",
       "  '',\n",
       "  'julia> d = D[\"death_rate\"];',\n",
       "  '',\n",
       "  \"julia> A = [b'; diagonal(1 .- d[1:end-1]) zeros(length(d)-1)];\",\n",
       "  'julia> x = D[\"population\"];',\n",
       "  '',\n",
       "  'julia> for k = 1:10',\n",
       "  '',\n",
       "  'global x',\n",
       "  '',\n",
       "  'x = A*x;',\n",
       "  '',\n",
       "  'end;',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> plot(x, legend=false, xlabel = \"Age\",',\n",
       "  '',\n",
       "  'ylabel = \"Population (millions)\")',\n",
       "  '',\n",
       "  'Note the keyword global in the for-loop. Without this statement, the scope of the',\n",
       "  'variable x created by the assignment x = A*x would be local to the for-loop, i.e.,',\n",
       "  'this variable does not exist outside the loop and is different from the x outside the',\n",
       "  'loop.',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section.9.2',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '9.3 Epidemic dynamics 75',\n",
       "  '',\n",
       "  '0 25 50 75 100',\n",
       "  '0',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  '3',\n",
       "  '',\n",
       "  '4',\n",
       "  '',\n",
       "  'Age',\n",
       "  '',\n",
       "  'P',\n",
       "  'op',\n",
       "  '',\n",
       "  'ul',\n",
       "  'at',\n",
       "  '',\n",
       "  'io',\n",
       "  'n ',\n",
       "  '',\n",
       "  '(m',\n",
       "  'ill',\n",
       "  '',\n",
       "  'io',\n",
       "  'ns',\n",
       "  '',\n",
       "  ')',\n",
       "  '',\n",
       "  'Figure 9.2 Predicted age distribution in the US in 2020.',\n",
       "  '',\n",
       "  '9.3 Epidemic dynamics',\n",
       "  '',\n",
       "  'Let’s implement the simulation of the epidemic dynamics from VMLS §9.3. The',\n",
       "  'plot is in figure 9.3.',\n",
       "  '',\n",
       "  'julia> T = 210;',\n",
       "  '',\n",
       "  'julia> A = [ 0.95 0.04 0 0 ; 0.05 0.85 0 0 ;',\n",
       "  '',\n",
       "  '0 0.10 1 0 ; 0 0.01 0 1 ];',\n",
       "  '',\n",
       "  'julia> x_1 = [1,0,0,0];',\n",
       "  '',\n",
       "  'julia> state_traj = [x_1 zeros(4,T-1) ]; # State trajectory',\n",
       "  '',\n",
       "  'julia> for t=1:T-1 # Dynamics recursion',\n",
       "  '',\n",
       "  'state_traj[:,t+1] = A*state_traj[:,t];',\n",
       "  '',\n",
       "  'end',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> plot(1:T, state_traj\\', xlabel = \"Time t\",',\n",
       "  'label = [\"Susceptible\", \"Infected\", \"Recovered\", \"Deceased\"])',\n",
       "  '',\n",
       "  '9.4 Motion of a mass',\n",
       "  '',\n",
       "  'Let’s simulate the discretized model of the motion of a mass in §9.4 of VMLS. See',\n",
       "  'figure 9.4.',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section.9.3',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section.9.4',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '76 9 Linear dynamical systems',\n",
       "  '',\n",
       "  '0 50 100 150 200',\n",
       "  '',\n",
       "  '0.00',\n",
       "  '',\n",
       "  '0.25',\n",
       "  '',\n",
       "  '0.50',\n",
       "  '',\n",
       "  '0.75',\n",
       "  '',\n",
       "  '1.00',\n",
       "  '',\n",
       "  'Time t',\n",
       "  '',\n",
       "  'Susceptible',\n",
       "  'Infected',\n",
       "  'Recovered',\n",
       "  'Deceased',\n",
       "  '',\n",
       "  'Figure 9.3 Simulation of epidemic dynamics.',\n",
       "  '',\n",
       "  'julia> h = 0.01; m = 1; eta = 1;',\n",
       "  '',\n",
       "  'julia> A = [ 1 h ; 0 1-h*eta/m ];',\n",
       "  '',\n",
       "  'julia> B = [ 0 ; h/m ];',\n",
       "  '',\n",
       "  'julia> x1 = [0,0];',\n",
       "  '',\n",
       "  'julia> K = 600; # simulate for K*h = 6 seconds',\n",
       "  '',\n",
       "  'julia> f = zeros(K); f[50:99] .= 1.0; f[100:139] .= -1.3;',\n",
       "  '',\n",
       "  'julia> X = [x1 zeros(2,K-1)];',\n",
       "  '',\n",
       "  'julia> for k=1:K-1',\n",
       "  '',\n",
       "  'X[:,k+1] = A* X[:,k] + B*f[k]',\n",
       "  '',\n",
       "  'end',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> plot(X[1,:], xlabel=\"k\", ylabel=\"Position\", legend=false )',\n",
       "  '',\n",
       "  'julia> plot(X[2,:], xlabel=\"k\", ylabel=\"Velocity\", legend=false )',\n",
       "  '',\n",
       "  '9.5 Supply chain dynamics',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '9.5 Supply chain dynamics 77',\n",
       "  '',\n",
       "  '0 100 200 300 400 500 600',\n",
       "  '',\n",
       "  '0.00',\n",
       "  '',\n",
       "  '0.05',\n",
       "  '',\n",
       "  '0.10',\n",
       "  '',\n",
       "  '0.15',\n",
       "  '',\n",
       "  'k',\n",
       "  '',\n",
       "  'P',\n",
       "  'os',\n",
       "  '',\n",
       "  'iti',\n",
       "  'on',\n",
       "  '',\n",
       "  '0 100 200 300 400 500 600',\n",
       "  '',\n",
       "  '-0.1',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '0.1',\n",
       "  '',\n",
       "  '0.2',\n",
       "  '',\n",
       "  '0.3',\n",
       "  '',\n",
       "  '0.4',\n",
       "  '',\n",
       "  'k',\n",
       "  '',\n",
       "  'V',\n",
       "  'el',\n",
       "  '',\n",
       "  'oc',\n",
       "  'ity',\n",
       "  '',\n",
       "  'Figure 9.4 Simulation of a mass moving along a line: position (top) and',\n",
       "  'velocity (bottom).',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '78 9 Linear dynamical systems',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Chapter 10',\n",
       "  '',\n",
       "  'Matrix multiplication',\n",
       "  '',\n",
       "  '10.1 Matrix-matrix multiplication',\n",
       "  '',\n",
       "  'In Julia the product of matrices A and B is obtained with A*B. We calculate the',\n",
       "  'matrix product on page 177 of VMLS.',\n",
       "  '',\n",
       "  'julia> A = [-1.5 3 2; 1 -1 0]',\n",
       "  '',\n",
       "  '2×3 Array{Float64,2}:',\n",
       "  '-1.5 3.0 2.0',\n",
       "  '',\n",
       "  '1.0 -1.0 0.0',\n",
       "  '',\n",
       "  'julia> B = [-1 -1; 0 -2; 1 0]',\n",
       "  '',\n",
       "  '3×2 Array{Int64,2}:',\n",
       "  '-1 -1',\n",
       "  '',\n",
       "  '0 -2',\n",
       "  '',\n",
       "  '1 0',\n",
       "  '',\n",
       "  'julia> C = A*B',\n",
       "  '',\n",
       "  '2×2 Array{Float64,2}:',\n",
       "  '3.5 -4.5',\n",
       "  '',\n",
       "  '-1.0 1.0',\n",
       "  '',\n",
       "  'Gram matrix. The Gram matrix of a matrix A is the matrix G = ATA. It is a',\n",
       "  'symmetric matrix and the i, j element Gij is the inner product of columns i and j',\n",
       "  'of A.',\n",
       "  '',\n",
       "  'julia> A = randn(10,3);',\n",
       "  '',\n",
       "  \"julia> G = A'*A\",\n",
       "  '3×3 Array{Float64,2}:',\n",
       "  '11.1364 -3.91865 -3.69057',\n",
       "  '',\n",
       "  '-3.91865 7.98358 2.4839',\n",
       "  '',\n",
       "  '-3.69057 2.4839 10.956',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section.10.1',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '80 10 Matrix multiplication',\n",
       "  '',\n",
       "  'julia> # Gii is norm of column i, squared',\n",
       "  '',\n",
       "  'julia> G[2,2]',\n",
       "  '',\n",
       "  '7.983579175590987',\n",
       "  '',\n",
       "  'julia> norm(A[:,2])^2',\n",
       "  '',\n",
       "  '7.983579175590987',\n",
       "  '',\n",
       "  'julia> # Gij is inner product of columns i and j',\n",
       "  '',\n",
       "  'julia> G[1,3]',\n",
       "  '',\n",
       "  '-3.6905664879621454',\n",
       "  '',\n",
       "  \"julia> A[:,1]'*A[:,3]\",\n",
       "  '-3.6905664879621463',\n",
       "  '',\n",
       "  'Complexity of matrix triple product. Let’s check the associative property, which',\n",
       "  'states that (AB)C = A(BC) for any m×n matrix A, any n×p matrix B, and any',\n",
       "  'p × q matrix B. At the same time we will see that the left-hand and right-hand',\n",
       "  'sides take very different amounts of time to compute.',\n",
       "  '',\n",
       "  'julia> m = 2000; n = 50; q = 2000; p = 2000;',\n",
       "  '',\n",
       "  'julia> A = randn(m,n); B = randn(n,p); C = randn(p,q);',\n",
       "  '',\n",
       "  'julia> @time LHS = (A*B)*C;',\n",
       "  '',\n",
       "  '0.819912 seconds (245.32 k allocations: 72.557 MiB, 13.47% gc time)',\n",
       "  '',\n",
       "  'julia> @time LHS = (A*B)*C;',\n",
       "  '',\n",
       "  '0.254107 seconds (8 allocations: 61.035 MiB, 20.85% gc time)',\n",
       "  '',\n",
       "  'julia> @time RHS = A*(B*C);',\n",
       "  '',\n",
       "  '0.030907 seconds (9 allocations: 31.281 MiB, 13.95% gc time)',\n",
       "  '',\n",
       "  'julia> @time RHS = A*(B*C);',\n",
       "  '',\n",
       "  '0.023507 seconds (8 allocations: 31.281 MiB, 3.13% gc time)',\n",
       "  '',\n",
       "  'julia> norm(LHS-RHS)',\n",
       "  '',\n",
       "  '5.334805188873507e-10',\n",
       "  '',\n",
       "  'julia> @time D = A*B*C; # evaluated as (A*B)*C or as A*(B*C)?',\n",
       "  '',\n",
       "  '0.220616 seconds (1.03 k allocations: 61.098 MiB, 5.09% gc time)',\n",
       "  '',\n",
       "  'We see that evaluating (A*B)*C takes around 10 times as much time as evaluating',\n",
       "  'A*(B*C), which is predicted from the complexities. In the last line we deduce that',\n",
       "  'A*B*C is evaluated left to right, as (A*B)*C. Note that for these particular matrices,',\n",
       "  'this is the (much) slower order to multiply the matrices.',\n",
       "  '',\n",
       "  '10.2 Composition of linear functions',\n",
       "  '',\n",
       "  'Second difference matrix. We compute the second difference matrix on page 184',\n",
       "  'of VMLS.',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.221',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '10.3 Matrix power 81',\n",
       "  '',\n",
       "  'julia> D(n) = [-eye(n-1) zeros(n-1)] + [zeros(n-1) eye(n-1)];',\n",
       "  '',\n",
       "  'julia> D(5)',\n",
       "  '',\n",
       "  '4×5 Array{Float64,2}:',\n",
       "  '-1.0 1.0 0.0 0.0 0.0',\n",
       "  '',\n",
       "  '0.0 -1.0 1.0 0.0 0.0',\n",
       "  '',\n",
       "  '0.0 0.0 -1.0 1.0 0.0',\n",
       "  '',\n",
       "  '0.0 0.0 0.0 -1.0 1.0',\n",
       "  '',\n",
       "  'julia> D(4)',\n",
       "  '',\n",
       "  '3×4 Array{Float64,2}:',\n",
       "  '-1.0 1.0 0.0 0.0',\n",
       "  '',\n",
       "  '0.0 -1.0 1.0 0.0',\n",
       "  '',\n",
       "  '0.0 0.0 -1.0 1.0',\n",
       "  '',\n",
       "  'julia> Delta = D(4)*D(5) # Second difference matrix',\n",
       "  '',\n",
       "  '3×5 Array{Float64,2}:',\n",
       "  '1.0 -2.0 1.0 0.0 0.0',\n",
       "  '',\n",
       "  '0.0 1.0 -2.0 1.0 0.0',\n",
       "  '',\n",
       "  '0.0 0.0 1.0 -2.0 1.0',\n",
       "  '',\n",
       "  '10.3 Matrix power',\n",
       "  '',\n",
       "  'The kth power of a square matrix A is denoted Ak. In Julia, this power is formed',\n",
       "  'using A^k.',\n",
       "  '',\n",
       "  'Let’s form the adjacency matrix of the directed graph on VMLS page 186. Then',\n",
       "  'let’s find out how many cycles of length 8 there are, starting from each node. (A',\n",
       "  'cycle is a path that starts and stops at the same node.)',\n",
       "  '',\n",
       "  'julia> A = [ 0 1 0 0 1; 1 0 1 0 0; 0 0 1 1 1; 1 0 0 0 0; 0 0 0 1 0]',\n",
       "  '',\n",
       "  '5×5 Array{Int64,2}:',\n",
       "  '0 1 0 0 1',\n",
       "  '',\n",
       "  '1 0 1 0 0',\n",
       "  '',\n",
       "  '0 0 1 1 1',\n",
       "  '',\n",
       "  '1 0 0 0 0',\n",
       "  '',\n",
       "  '0 0 0 1 0',\n",
       "  '',\n",
       "  'julia> A^2',\n",
       "  '',\n",
       "  '5×5 Array{Int64,2}:',\n",
       "  '1 0 1 1 0',\n",
       "  '',\n",
       "  '0 1 1 1 2',\n",
       "  '',\n",
       "  '1 0 1 2 1',\n",
       "  '',\n",
       "  '0 1 0 0 1',\n",
       "  '',\n",
       "  '1 0 0 0 0',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.226',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '82 10 Matrix multiplication',\n",
       "  '',\n",
       "  'julia> A^8',\n",
       "  '',\n",
       "  '5×5 Array{Int64,2}:',\n",
       "  '18 11 15 20 20',\n",
       "  '',\n",
       "  '25 14 21 28 26',\n",
       "  '',\n",
       "  '24 14 20 27 26',\n",
       "  '',\n",
       "  '11 6 9 12 11',\n",
       "  '',\n",
       "  '6 4 5 7 7',\n",
       "  '',\n",
       "  'julia> number_of_cycles = diag(A^8)',\n",
       "  '',\n",
       "  '5-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '18',\n",
       "  '',\n",
       "  '14',\n",
       "  '',\n",
       "  '20',\n",
       "  '',\n",
       "  '12',\n",
       "  '',\n",
       "  '7',\n",
       "  '',\n",
       "  'Population dynamics. Let’s write the code for figure 10.2 in VMLS, which plots',\n",
       "  'the contribution factor to the total US population in 2020 (ignoring immigration),',\n",
       "  'for each age in 2010. The Julia plot is in figure 10.1. We can see that, not',\n",
       "  'surprisingly, 20–25 year olds have the highest contributing factor, around 1.5. This',\n",
       "  'means that on average, each 20-25 year old in 2010 will be responsible for around',\n",
       "  '1.5 people in 2020. This takes into account any children they may have before',\n",
       "  'then, and (as a very small effect) the few of them who will no longer be with us in',\n",
       "  '2020.',\n",
       "  '',\n",
       "  'julia> D = population_data();',\n",
       "  '',\n",
       "  'julia> b = D[\"birth_rate\"];',\n",
       "  '',\n",
       "  'julia> d = D[\"death_rate\"];',\n",
       "  '',\n",
       "  'julia> # Dynamics matrix for populaion dynamics',\n",
       "  '',\n",
       "  \"julia> A = [b'; diagonal(1 .- d[1:end-1]) zeros(length(d)-1)];\",\n",
       "  'julia> # Contribution factor to total poulation in 2020',\n",
       "  '',\n",
       "  'julia> # from each age in 2010',\n",
       "  '',\n",
       "  \"julia> cf = ones(100)'*(A^10); # Contribution factor\",\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> plot(cf\\', legend = false, xlabel = \"Age\", ylabel = \"Factor\")',\n",
       "  '',\n",
       "  '10.4 QR factorization',\n",
       "  '',\n",
       "  'In Julia, the QR factorization of a matrix A can be found using qr(A), which',\n",
       "  'returns a tuple with the Q and R factors. However the matrix Q is not returned as',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.10.2',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '10.4 QR factorization 83',\n",
       "  '',\n",
       "  '0 25 50 75 100',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '0.5',\n",
       "  '',\n",
       "  '1.0',\n",
       "  '',\n",
       "  '1.5',\n",
       "  '',\n",
       "  'Age',\n",
       "  '',\n",
       "  'F',\n",
       "  'ac',\n",
       "  '',\n",
       "  'to',\n",
       "  'r',\n",
       "  '',\n",
       "  'Figure 10.1 Contribution factor per age in 2010 to the total population in',\n",
       "  '2020. The value for age i− 1 is the ith component of the row vector 1TA10.',\n",
       "  '',\n",
       "  'an array, but in a special compact format. It can be converted to a regular matrix',\n",
       "  'variable using the command Matrix(Q). Hence, the QR factorization as defined in',\n",
       "  'VMLS is computed by a sequence of two commands:',\n",
       "  '',\n",
       "  'julia> Q, R = qr(A);',\n",
       "  '',\n",
       "  'julia> Q = Matrix(Q);',\n",
       "  '',\n",
       "  'The following example also illustates a second, but minor difference with the',\n",
       "  'VMLS definition. The R factor computed by Julia may have negative elements on',\n",
       "  'the diagonal, as opposed to only positive elements if we follow the definition used',\n",
       "  'in VMLS. The two definitions are equivalent, because if Rii is negative, one can',\n",
       "  'change the sign of the ith row of R and the ith column of Q, to get an equivalent',\n",
       "  'factorization with Rii > 0. However this step is not needed in practice, since',\n",
       "  'negative elements on the diagonal do not pose any problem in applications of the',\n",
       "  'QR factorization.',\n",
       "  '',\n",
       "  'julia> A = randn(6,4);',\n",
       "  '',\n",
       "  'julia> Q, R = qr(A);',\n",
       "  '',\n",
       "  'julia> R',\n",
       "  '',\n",
       "  '4×4 Array{Float64,2}:',\n",
       "  '1.36483 -1.21281 -0.470052 -1.40011',\n",
       "  '',\n",
       "  '0.0 -2.01191 -2.92458 -0.802368',\n",
       "  '',\n",
       "  '0.0 0.0 -1.94759 0.84228',\n",
       "  '',\n",
       "  '0.0 0.0 0.0 1.19766',\n",
       "  '',\n",
       "  'julia> Q = Matrix(Q)',\n",
       "  '',\n",
       "  '6×4 Array{Float64,2}:',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '84 10 Matrix multiplication',\n",
       "  '',\n",
       "  '-0.454997 -0.442971 -0.00869016 0.121067',\n",
       "  '',\n",
       "  '-0.577229 -0.301916 0.349002 -0.370212',\n",
       "  '',\n",
       "  '0.00581297 -0.159604 -0.647797 0.28373',\n",
       "  '',\n",
       "  '0.221266 0.388812 0.223813 -0.46152',\n",
       "  '',\n",
       "  '-0.550089 0.546873 -0.507827 -0.30155',\n",
       "  '',\n",
       "  '-0.328929 0.48673 0.387945 0.681065',\n",
       "  '',\n",
       "  'julia> norm(Q*R-A)',\n",
       "  '',\n",
       "  '1.0046789954275695e-15',\n",
       "  '',\n",
       "  \"julia> Q'*Q\",\n",
       "  '4×4 Array{Float64,2}:',\n",
       "  '1.0 0.0 -2.77556e-17 2.77556e-17',\n",
       "  '',\n",
       "  '0.0 1.0 -8.32667e-17 -1.66533e-16',\n",
       "  '',\n",
       "  '-2.77556e-17 -8.32667e-17 1.0 5.55112e-17',\n",
       "  '',\n",
       "  '2.77556e-17 -1.66533e-16 5.55112e-17 1.0',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Chapter 11',\n",
       "  '',\n",
       "  'Matrix inverses',\n",
       "  '',\n",
       "  '11.1 Left and right inverses',\n",
       "  '',\n",
       "  'We’ll see later how to find a left or right inverse, when one exists.',\n",
       "  '',\n",
       "  'julia> A = [-3 -4; 4 6; 1 1]',\n",
       "  '',\n",
       "  '3×2 Array{Int64,2}:',\n",
       "  '-3 -4',\n",
       "  '',\n",
       "  '4 6',\n",
       "  '',\n",
       "  '1 1',\n",
       "  '',\n",
       "  'julia> B = [-11 -10 16; 7 8 -11]/9 # A left inverse of A',\n",
       "  '',\n",
       "  '2×3 Array{Float64,2}:',\n",
       "  '-1.22222 -1.11111 1.77778',\n",
       "  '',\n",
       "  '0.777778 0.888889 -1.22222',\n",
       "  '',\n",
       "  'julia> C = [0 -1 6; 0 1 -4]/2 # Another left inverse of A',\n",
       "  '',\n",
       "  '2×3 Array{Float64,2}:',\n",
       "  '0.0 -0.5 3.0',\n",
       "  '',\n",
       "  '0.0 0.5 -2.0',\n",
       "  '',\n",
       "  'julia> # Let’s check',\n",
       "  '',\n",
       "  'julia> B*A',\n",
       "  '',\n",
       "  '2×2 Array{Float64,2}:',\n",
       "  '1.0 0.0',\n",
       "  '',\n",
       "  '-4.44089e-16 1.0',\n",
       "  '',\n",
       "  'julia> C*A',\n",
       "  '',\n",
       "  '2×2 Array{Float64,2}:',\n",
       "  '1.0 0.0',\n",
       "  '',\n",
       "  '0.0 1.0',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '86 11 Matrix inverses',\n",
       "  '',\n",
       "  '11.2 Inverse',\n",
       "  '',\n",
       "  'If A is invertible, its inverse is given by inv(A) (and also A^-1). You’ll get an error',\n",
       "  'if A is not invertible, or not square.',\n",
       "  '',\n",
       "  'julia> A = [1 -2 3; 0 2 2; -4 -4 -4]',\n",
       "  '',\n",
       "  '3×3 Array{Int64,2}:',\n",
       "  '1 -2 3',\n",
       "  '',\n",
       "  '0 2 2',\n",
       "  '',\n",
       "  '-4 -4 -4',\n",
       "  '',\n",
       "  'julia> B = inv(A)',\n",
       "  '',\n",
       "  '3×3 Array{Float64,2}:',\n",
       "  '0.0 -0.5 -0.25',\n",
       "  '',\n",
       "  '-0.2 0.2 -0.05',\n",
       "  '',\n",
       "  '0.2 0.3 0.05',\n",
       "  '',\n",
       "  'julia> B*A',\n",
       "  '',\n",
       "  '3×3 Array{Float64,2}:',\n",
       "  '1.0 0.0 0.0',\n",
       "  '',\n",
       "  '0.0 1.0 2.77556e-17',\n",
       "  '',\n",
       "  '0.0 5.55112e-17 1.0',\n",
       "  '',\n",
       "  'julia> A*B',\n",
       "  '',\n",
       "  '3×3 Array{Float64,2}:',\n",
       "  '1.0 1.11022e-16 0.0',\n",
       "  '',\n",
       "  '5.55112e-17 1.0 1.38778e-17',\n",
       "  '',\n",
       "  '-1.11022e-16 -2.22045e-16 1.0',\n",
       "  '',\n",
       "  'Dual basis. The next example illustrates the dual basis provided by the rows of',\n",
       "  'the inverse B = A−1. We calculate the expansion',\n",
       "  '',\n",
       "  'x = (bT1 x)a1 + · · ·+ (bTnx)an',\n",
       "  '',\n",
       "  'for a 3× 3 example (see page 205 of VMLS).',\n",
       "  '',\n",
       "  'julia> A = [ 1 0 1; 4 -3 -4; 1 -1 -2]',\n",
       "  '',\n",
       "  '3×3 Array{Int64,2}:',\n",
       "  '1 0 1',\n",
       "  '',\n",
       "  '4 -3 -4',\n",
       "  '',\n",
       "  '1 -1 -2',\n",
       "  '',\n",
       "  'julia> B = inv(A)',\n",
       "  '',\n",
       "  '3×3 Array{Float64,2}:',\n",
       "  '2.0 -1.0 3.0',\n",
       "  '',\n",
       "  '4.0 -3.0 8.0',\n",
       "  '',\n",
       "  '-1.0 1.0 -3.0',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.249',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '11.3 Solving linear equations 87',\n",
       "  '',\n",
       "  'julia> x = [ 0.2, -0.3, 1.2]',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.2',\n",
       "  '',\n",
       "  '-0.3',\n",
       "  '',\n",
       "  '1.2',\n",
       "  '',\n",
       "  \"julia> rhs = (B[1,:]'*x) * A[:,1] + (B[2,:]'*x) * A[:,2]\",\n",
       "  \"+ (B[3,:]'*x) * A[:,3]\",\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.2',\n",
       "  '',\n",
       "  '-0.3',\n",
       "  '',\n",
       "  '1.2',\n",
       "  '',\n",
       "  'Inverse via QR factorization. The inverse of a matrix A can be computed from',\n",
       "  'its QR factorization A = QR via the formula A−1 = R−1QT .',\n",
       "  '',\n",
       "  'julia> A = randn(3,3);',\n",
       "  '',\n",
       "  'julia> inv(A)',\n",
       "  '',\n",
       "  '3×3 Array{Float64,2}:',\n",
       "  '-0.321679 0.323945 -0.347063',\n",
       "  '',\n",
       "  '-1.0735 -5.03083 -3.24503',\n",
       "  '',\n",
       "  '-1.17582 -2.68161 -1.8496',\n",
       "  '',\n",
       "  'julia> Q, R = qr(A);',\n",
       "  '',\n",
       "  'julia> Q = Matrix(Q);',\n",
       "  '',\n",
       "  \"julia> inv(R)*Q'\",\n",
       "  '3×3 Array{Float64,2}:',\n",
       "  '-0.321679 0.323945 -0.347063',\n",
       "  '',\n",
       "  '-1.0735 -5.03083 -3.24503',\n",
       "  '',\n",
       "  '-1.17582 -2.68161 -1.8496',\n",
       "  '',\n",
       "  '11.3 Solving linear equations',\n",
       "  '',\n",
       "  'Back substitution. Let’s first implement back substitution (VMLS Algorithm 11.1)',\n",
       "  'in Julia, and check it. You won’t need this function, since Julia has a better im-',\n",
       "  'plementation of it built in (via the backslash operation discussed below). We give',\n",
       "  'it here only to demonstrate that it works.',\n",
       "  '',\n",
       "  'julia> function back_subst(R,b)',\n",
       "  '',\n",
       "  'n = length(b)',\n",
       "  '',\n",
       "  'x = zeros(n)',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#algorithmctr.11.1',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '88 11 Matrix inverses',\n",
       "  '',\n",
       "  'for i=n:-1:1',\n",
       "  '',\n",
       "  \"x[i] = (b[i] - R[i,i+1:n]'*x[i+1:n]) / R[i,i]\",\n",
       "  'end',\n",
       "  '',\n",
       "  'return x',\n",
       "  '',\n",
       "  'end;',\n",
       "  '',\n",
       "  'julia> R = triu(randn(4,4)) # Random 4x4 upper triangular matrix',\n",
       "  '',\n",
       "  '4×4 Array{Float64,2}:',\n",
       "  '-0.498881 -0.880538 -0.745078 0.125678',\n",
       "  '',\n",
       "  '0.0 -0.922477 -0.00673699 0.30122',\n",
       "  '',\n",
       "  '0.0 0.0 -0.283035 -0.0184466',\n",
       "  '',\n",
       "  '0.0 0.0 0.0 -2.01396',\n",
       "  '',\n",
       "  'julia> b = rand(4);',\n",
       "  '',\n",
       "  'julia> x = back_subst(R,b);',\n",
       "  '',\n",
       "  'julia> norm(R*x-b)',\n",
       "  '',\n",
       "  '2.220446049250313e-16',\n",
       "  '',\n",
       "  'The function triu gives the upper triangular part of a matrix, i.e., it zeros out the',\n",
       "  'entries below the diagonal.',\n",
       "  '',\n",
       "  'Backslash notation. The Julia command for solving a set of linear equations',\n",
       "  '',\n",
       "  'Ax = b',\n",
       "  '',\n",
       "  'is x=A\\\\b. This is faster than x=inv(A)*b, which first computes the inverse of A',\n",
       "  'and then multiplies it with b.',\n",
       "  '',\n",
       "  'julia> n = 5000;',\n",
       "  '',\n",
       "  'julia> A = randn(n,n); b = randn(n); # random set of equations',\n",
       "  '',\n",
       "  'julia> @time x1 = A\\\\b;',\n",
       "  '',\n",
       "  '1.422173 seconds (14 allocations: 190.812 MiB, 0.73% gc time)',\n",
       "  '',\n",
       "  'julia> norm(b-A*x1)',\n",
       "  '',\n",
       "  '3.8666263141510634e-9',\n",
       "  '',\n",
       "  'julia> @time x2 = inv(A)*b;',\n",
       "  '',\n",
       "  '4.550091 seconds (21 allocations: 384.026 MiB, 1.66% gc time)',\n",
       "  '',\n",
       "  'julia> norm(b-A*x2)',\n",
       "  '',\n",
       "  '1.682485195063787e-8',\n",
       "  '',\n",
       "  'Julia chooses a suitable algorithm for solving the equation after checking the',\n",
       "  'properties of A. For example, it will use back substitution if A is lower triangular.',\n",
       "  'This explains the result in the following timing experiment.',\n",
       "  '',\n",
       "  'julia> n = 5000;',\n",
       "  '',\n",
       "  'julia> b = randn(n);',\n",
       "  '',\n",
       "  'julia> A = tril(randn(n,n)); # random lower triangular matrix',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '11.3 Solving linear equations 89',\n",
       "  '',\n",
       "  'julia> @time x = A\\\\b;',\n",
       "  '',\n",
       "  '0.042580 seconds (7 allocations: 39.313 KiB)',\n",
       "  '',\n",
       "  'julia> A = randn(n,n); # random square matrix',\n",
       "  '',\n",
       "  'julia> @time x = A\\\\b;',\n",
       "  '',\n",
       "  '1.289663 seconds (14 allocations: 190.812 MiB, 0.91% gc time)',\n",
       "  '',\n",
       "  'julia> n = 10000;',\n",
       "  '',\n",
       "  'julia> b = randn(n);',\n",
       "  '',\n",
       "  'julia> A = tril(randn(n,n)); # random lower triangular matrix',\n",
       "  '',\n",
       "  'julia> @time x = A\\\\b;',\n",
       "  '',\n",
       "  '0.157043 seconds (7 allocations: 78.375 KiB)',\n",
       "  '',\n",
       "  'julia> A = randn(n,n); # random square matrix',\n",
       "  '',\n",
       "  'julia> @time x = A\\\\b;',\n",
       "  '',\n",
       "  '9.008604 seconds (14 allocations: 763.093 MiB, 0.41% gc time)',\n",
       "  '',\n",
       "  'When we double the size from n = 5000 to n = 10000, the solution time for the',\n",
       "  'triangular equation increases from 0.04 seconds to 1.29 seconds. This is a factor of',\n",
       "  'about four, consistent with the n2 complexity of backsubstitution. For the general',\n",
       "  'square system, the solution times increases from 0.16 seconds to 9.01 seconds, i.e.,',\n",
       "  'a factor of roughly eight, as we would expect given the order n3 complexity.',\n",
       "  '',\n",
       "  'Factor-solve methods for multiple right-hand sides. A linear equation is solved',\n",
       "  'by first factorizing A and then solving several simpler equations with the factors',\n",
       "  'of A. This is referred to as a factor-solve scheme. An important application is the',\n",
       "  'solution of multiple linear equations with the same coefficient matrix and different',\n",
       "  'right-hand sides.',\n",
       "  '',\n",
       "  'julia> n = 5000;',\n",
       "  '',\n",
       "  'julia> A = randn(n,n); B = randn(n,2);',\n",
       "  '',\n",
       "  'julia> # Solve with right-hand side B[:,1]',\n",
       "  '',\n",
       "  'julia> @time x1 = A \\\\ B[:,1];',\n",
       "  '',\n",
       "  '1.501368 seconds (17 allocations: 190.850 MiB, 0.79% gc time)',\n",
       "  '',\n",
       "  'julia> # Solve with right-hand side B[:,2]',\n",
       "  '',\n",
       "  'julia> @time x2 = A \\\\ B[:,2];',\n",
       "  '',\n",
       "  '1.388827 seconds (17 allocations: 190.850 MiB, 0.81% gc time)',\n",
       "  '',\n",
       "  'julia> # Naive approach for solving A*X = B',\n",
       "  '',\n",
       "  'julia> @time X = [ A\\\\B[:,1] A\\\\B[:,2] ];',\n",
       "  '',\n",
       "  '2.617617 seconds (35 allocations: 381.776 MiB, 3.19% gc time)',\n",
       "  '',\n",
       "  'julia> # Factor-solve approach',\n",
       "  '',\n",
       "  'julia> @time X = A \\\\ B;',\n",
       "  '',\n",
       "  '1.418451 seconds (83.37 k allocations: 194.881 MiB, 0.58% gc time)',\n",
       "  '',\n",
       "  'The factor-solve approach finds the solutions for the two right-hand sides in roughly',\n",
       "  'the same time as the solution for one right-hand side. The solution time with the',\n",
       "  'näıve approach is twice the time for one right-hand side.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '90 11 Matrix inverses',\n",
       "  '',\n",
       "  '11.4 Examples',\n",
       "  '',\n",
       "  'Polynomial interpolation. We compute the interpolating polynomials in Fig-',\n",
       "  'ure 11.1 of VMLS. The following code uses the functions vandermonde and linspace',\n",
       "  'from the VMLS package. (Th function linspace(a,b,n) returns a vector with n',\n",
       "  'equally spaced numbers in the interval [a, b].) The result is shown in Figure 11.1.',\n",
       "  '',\n",
       "  'julia> t = [ -1.1, -0.4, 0.2, 0.8 ];',\n",
       "  '',\n",
       "  'julia> A = vandermonde(t, 4)',\n",
       "  '',\n",
       "  '4×4 Array{Float64,2}:',\n",
       "  '1.0 -1.1 1.21 -1.331',\n",
       "  '',\n",
       "  '1.0 -0.4 0.16 -0.064',\n",
       "  '',\n",
       "  '1.0 0.2 0.04 0.008',\n",
       "  '',\n",
       "  '1.0 0.8 0.64 0.512',\n",
       "  '',\n",
       "  'julia> b1 = [ -1.0, 1.3, 1.0, 0.5 ];',\n",
       "  '',\n",
       "  'julia> c1 = A \\\\ b1',\n",
       "  '',\n",
       "  '4-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '1.21096',\n",
       "  '',\n",
       "  '-0.888311',\n",
       "  '',\n",
       "  '-1.10967',\n",
       "  '',\n",
       "  '1.38648',\n",
       "  '',\n",
       "  'julia> b2 = [ 1.0, 0.0, -2.0, 0 ];',\n",
       "  '',\n",
       "  'julia> c2 = A \\\\ b2',\n",
       "  '',\n",
       "  '4-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '-1.54129',\n",
       "  '',\n",
       "  '-3.10905',\n",
       "  '',\n",
       "  '3.33847',\n",
       "  '',\n",
       "  '3.69514',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> ts = linspace(-1.2, 1.2, 1000);',\n",
       "  '',\n",
       "  'julia> p1 = c1[1] .+ c1[2]*ts + c1[3]*ts.^2 + c1[4]*ts.^3;',\n",
       "  '',\n",
       "  'julia> plot(ts, p1)',\n",
       "  '',\n",
       "  'julia> scatter!(t, b1)',\n",
       "  '',\n",
       "  'julia> p2 = c2[1] .+ c2[2]*ts + c2[3]*ts.^2 + c2[4]*ts.^3;',\n",
       "  '',\n",
       "  'julia> plot!(ts, p2)',\n",
       "  '',\n",
       "  'julia> scatter!(t, b2, maker = :square)',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.11.1',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '11.5 Pseudo-inverse 91',\n",
       "  '',\n",
       "  '-1.0 -0.5 0.0 0.5 1.0',\n",
       "  '',\n",
       "  '-2',\n",
       "  '',\n",
       "  '-1',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  'Figure 11.1 Cubic interpolants through two sets of points, shown as circles',\n",
       "  'and squares.',\n",
       "  '',\n",
       "  '11.5 Pseudo-inverse',\n",
       "  '',\n",
       "  'In Julia, the pseudo-inverse of a matrix A is obtained with pinv(A). We compute',\n",
       "  'the pseudo-inverse for the example on page 216 of VMLS using the pinv function,',\n",
       "  'and via the formula A† = R−1QT , where A = QR is the QR factorization of A.',\n",
       "  '',\n",
       "  'julia> A = [-3 -4; 4 6; 1 1]',\n",
       "  '',\n",
       "  '3×2 Array{Int64,2}:',\n",
       "  '-3 -4',\n",
       "  '',\n",
       "  '4 6',\n",
       "  '',\n",
       "  '1 1',\n",
       "  '',\n",
       "  'julia> pinv(A)',\n",
       "  '',\n",
       "  '2×3 Array{Float64,2}:',\n",
       "  '-1.22222 -1.11111 1.77778',\n",
       "  '',\n",
       "  '0.777778 0.888889 -1.22222',\n",
       "  '',\n",
       "  'julia> Q, R = qr(A);',\n",
       "  '',\n",
       "  'julia> Q = Matrix(Q)',\n",
       "  '',\n",
       "  '3×2 Array{Float64,2}:',\n",
       "  '-0.588348 -0.457604',\n",
       "  '',\n",
       "  '0.784465 -0.522976',\n",
       "  '',\n",
       "  '0.196116 0.719092',\n",
       "  '',\n",
       "  'julia> R',\n",
       "  '',\n",
       "  '2×2 Array{Float64,2}:',\n",
       "  '5.09902 7.2563',\n",
       "  '',\n",
       "  '0.0 -0.588348',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.270',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '92 11 Matrix inverses',\n",
       "  '',\n",
       "  \"julia> R \\\\ Q' # pseudo-inverse from QR factors\",\n",
       "  '2×3 Array{Float64,2}:',\n",
       "  '-1.22222 -1.11111 1.77778',\n",
       "  '',\n",
       "  '0.777778 0.888889 -1.22222',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Chapter 12',\n",
       "  '',\n",
       "  'Least squares',\n",
       "  '',\n",
       "  '12.1 Least squares problem',\n",
       "  '',\n",
       "  'We take the small least squares problem of Figure 12.1 in VMLS and check that',\n",
       "  '‖Ax̂− b‖ is less than ‖Ax− b‖ for some other value of x.',\n",
       "  '',\n",
       "  'julia> A = [ 2 0 ; -1 1 ; 0 2 ]',\n",
       "  '',\n",
       "  '3×2 Array{Int64,2}:',\n",
       "  '2 0',\n",
       "  '',\n",
       "  '-1 1',\n",
       "  '',\n",
       "  '0 2',\n",
       "  '',\n",
       "  'julia> b = [ 1, 0, -1 ]',\n",
       "  '',\n",
       "  '3-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  '-1',\n",
       "  '',\n",
       "  'julia> xhat = [ 1/3, -1/3 ]',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.333333',\n",
       "  '',\n",
       "  '-0.333333',\n",
       "  '',\n",
       "  'julia> rhat = A*xhat -b',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '-0.333333',\n",
       "  '',\n",
       "  '-0.666667',\n",
       "  '',\n",
       "  '0.333333',\n",
       "  '',\n",
       "  'julia> norm(rhat)',\n",
       "  '',\n",
       "  '0.816496580927726',\n",
       "  '',\n",
       "  'julia> x = [ 1/2, -1/2 ]',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.5',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.12.1',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '94 12 Least squares',\n",
       "  '',\n",
       "  '-0.5',\n",
       "  '',\n",
       "  'julia> r = A*x -b',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '-1.0',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  'julia> norm(r)',\n",
       "  '',\n",
       "  '1.0',\n",
       "  '',\n",
       "  '12.2 Solution',\n",
       "  '',\n",
       "  'Least squares solution formula. Let’s check the solution formulas (12.5) and',\n",
       "  '(12.6) in VMLS,',\n",
       "  '',\n",
       "  'x̂ = (ATA)−1AT b = A†b',\n",
       "  '',\n",
       "  'for the small example of Figure 12.1) (where x̂ = (1/3, 1/3)).',\n",
       "  '',\n",
       "  \"julia> inv(A'*A)*A'*b\",\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.333333',\n",
       "  '',\n",
       "  '-0.333333',\n",
       "  '',\n",
       "  'julia> pinv(A)*b',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.333333',\n",
       "  '',\n",
       "  '-0.333333',\n",
       "  '',\n",
       "  \"julia> (A'*A)*xhat - A'*b # Check that normal equations hold\",\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '-8.88178e-16',\n",
       "  '',\n",
       "  'Orthogonality principle. Let’s check the orthogonality principle (12.9), for the',\n",
       "  'same example.',\n",
       "  '',\n",
       "  'julia> z = [-1.1, 2.3];',\n",
       "  '',\n",
       "  \"julia> (A*z)'*rhat\",\n",
       "  '2.220446049250313e-16',\n",
       "  '',\n",
       "  'julia> z = [5.3, -1.2];',\n",
       "  '',\n",
       "  \"julia> (A*z)'*rhat\",\n",
       "  '-6.661338147750939e-16',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#equation.12.2.5',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#equation.12.2.6',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.12.1',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#equation.12.2.9',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '12.3 Solving least squares problems 95',\n",
       "  '',\n",
       "  '12.3 Solving least squares problems',\n",
       "  '',\n",
       "  'Julia uses the backslash operator to denote the least squares approximate solu-',\n",
       "  'tion: xhat = A\\\\b. (The same operator is used to solve square systems of linear',\n",
       "  'equations, and we will see more uses of it in later chapters.)',\n",
       "  '',\n",
       "  'julia> A = randn(100,20); b = randn(100);',\n",
       "  '',\n",
       "  'julia> x1 = A\\\\b; # Least squares using backslash operator',\n",
       "  '',\n",
       "  \"julia> x2 = inv(A'*A)*(A'*b); # Using formula\",\n",
       "  'julia> x3 = pinv(A)*b; # Using pseudo-inverse',\n",
       "  '',\n",
       "  'julia> Q, R = qr(A);',\n",
       "  '',\n",
       "  'julia> Q = Matrix(Q);',\n",
       "  '',\n",
       "  \"julia> x4 = R\\\\(Q'*b); # Using QR factorization\",\n",
       "  'julia> norm(x1-x2)',\n",
       "  '',\n",
       "  '4.258136640215341e-16',\n",
       "  '',\n",
       "  'julia> norm(x2-x3)',\n",
       "  '',\n",
       "  '1.3328728382991758e-15',\n",
       "  '',\n",
       "  'julia> norm(x3-x4)',\n",
       "  '',\n",
       "  '1.3507615689695538e-15',\n",
       "  '',\n",
       "  'Complexity. The complexity of solving the least squares problem with m × n',\n",
       "  'matrix A is around 2mn2 flops. Let’s check this in Julia by solving a few least',\n",
       "  'squares problems of different dimensions.',\n",
       "  '',\n",
       "  'julia> m = 2000; n = 500;',\n",
       "  '',\n",
       "  'julia> A = randn(m,n); b = randn(m);',\n",
       "  '',\n",
       "  'julia> @time x = A\\\\b;',\n",
       "  '',\n",
       "  '0.190497 seconds (4.07 k allocations: 12.031 MiB, 28.89% gc time)',\n",
       "  '',\n",
       "  'julia> @time x = A\\\\b;',\n",
       "  '',\n",
       "  '0.120246 seconds (4.07 k allocations: 12.031 MiB)',\n",
       "  '',\n",
       "  'julia> m = 4000; n = 500;',\n",
       "  '',\n",
       "  'julia> A = randn(m,n); b = randn(m);',\n",
       "  '',\n",
       "  'julia> @time x = A\\\\b;',\n",
       "  '',\n",
       "  '0.248510 seconds (4.07 k allocations: 19.675 MiB)',\n",
       "  '',\n",
       "  'julia> m = 2000; n = 1000;',\n",
       "  '',\n",
       "  'julia> A = randn(m,n); b = randn(m);',\n",
       "  '',\n",
       "  'julia> @time x = A\\\\b;',\n",
       "  '',\n",
       "  '0.418181 seconds (8.07 k allocations: 31.608 MiB, 1.66% gc time)',\n",
       "  '',\n",
       "  'We can see that doubling m approximately doubles the computation time, and',\n",
       "  'doubling n increases it by around a factor of four. The times above can be used to',\n",
       "  'guess the speed of the computer on which it was carried out. For example, using',\n",
       "  'the last problem solved, the number of flops is around 2mn2 = 4 · 109, and it took',\n",
       "  'around 0.4 seconds. This suggests a speed of around 10 Gflop/sec.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '96 12 Least squares',\n",
       "  '',\n",
       "  'Matrix least squares. Let’s solve multiple least squares problems with the same',\n",
       "  'matrix A and different vectors b.',\n",
       "  '',\n",
       "  'julia> A = randn(1000,100); B = randn(1000,10);',\n",
       "  '',\n",
       "  'julia> X = A\\\\B;',\n",
       "  '',\n",
       "  'julia> # Check that third column of X is least squares solution',\n",
       "  '',\n",
       "  'julia> # with third column of B',\n",
       "  '',\n",
       "  'julia> x3 = A\\\\B[:,3];',\n",
       "  '',\n",
       "  'julia> norm(X[:,3]-x3)',\n",
       "  '',\n",
       "  '1.1037090583270415e-16',\n",
       "  '',\n",
       "  '12.4 Examples',\n",
       "  '',\n",
       "  'Advertising purchases. We work out the solution of the optimal advertising pur-',\n",
       "  'chase problem on page 234 of VMLS.',\n",
       "  '',\n",
       "  'julia> R = [ 0.97 1.86 0.41;',\n",
       "  '',\n",
       "  '1.23 2.18 0.53;',\n",
       "  '',\n",
       "  '0.80 1.24 0.62;',\n",
       "  '',\n",
       "  '1.29 0.98 0.51;',\n",
       "  '',\n",
       "  '1.10 1.23 0.69;',\n",
       "  '',\n",
       "  '0.67 0.34 0.54;',\n",
       "  '',\n",
       "  '0.87 0.26 0.62;',\n",
       "  '',\n",
       "  '1.10 0.16 0.48;',\n",
       "  '',\n",
       "  '1.92 0.22 0.71;',\n",
       "  '',\n",
       "  '1.29 0.12 0.62];',\n",
       "  '',\n",
       "  'julia> m, n = size(R);',\n",
       "  '',\n",
       "  'julia> vdes = 1e3 * ones(m);',\n",
       "  '',\n",
       "  'julia> s = R \\\\ vdes',\n",
       "  '',\n",
       "  '31 Array{Float64,2}:',\n",
       "  '',\n",
       "  '62.0766',\n",
       "  '',\n",
       "  '99.985',\n",
       "  '',\n",
       "  '1442.84',\n",
       "  '',\n",
       "  'julia> rms(R*s - vdes)',\n",
       "  '',\n",
       "  '132.63819026326527',\n",
       "  '',\n",
       "  'Illumination. The following code constructs and solves the illumination problem',\n",
       "  'on page 234, and plots two histograms with the pixel intensity distributions (Fig-',\n",
       "  'ure 12.1).',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.284',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.285',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '12.4 Examples 97',\n",
       "  '',\n",
       "  'julia> n = 10; # number of lamps',\n",
       "  '',\n",
       "  'julia> lamps = [ # x, y positions of lamps and height above floor',\n",
       "  '',\n",
       "  '4.1 20.4 4;',\n",
       "  '',\n",
       "  '14.1 21.3 3.5;',\n",
       "  '',\n",
       "  '22.6 17.1 6;',\n",
       "  '',\n",
       "  '5.5 12.3 4.0;',\n",
       "  '',\n",
       "  '12.2 9.7 4.0;',\n",
       "  '',\n",
       "  '15.3 13.8 6;',\n",
       "  '',\n",
       "  '21.3 10.5 5.5;',\n",
       "  '',\n",
       "  '3.9 3.3 5.0;',\n",
       "  '',\n",
       "  '13.1 4.3 5.0;',\n",
       "  '',\n",
       "  '20.3 4.2 4.5 ];',\n",
       "  '',\n",
       "  'julia> N = 25; # grid size',\n",
       "  '',\n",
       "  'julia> m = N*N; # number of pixels',\n",
       "  '',\n",
       "  'julia> # construct m x 2 matrix with coordinates of pixel centers',\n",
       "  '',\n",
       "  'julia> pixels = hcat(',\n",
       "  '',\n",
       "  'reshape( collect(0.5: 1 : N) * ones(1,N), m, 1),',\n",
       "  '',\n",
       "  'reshape( ones(N,1) * collect(0.5: 1 : N)’, m, 1) );',\n",
       "  '',\n",
       "  'julia> # The m x n matrix A maps lamp powers to pixel intensities.',\n",
       "  '',\n",
       "  'julia> # A[i,j] is inversely proportional to the squared distance of',\n",
       "  '',\n",
       "  'julia> # lamp j to pixel i.',\n",
       "  '',\n",
       "  'julia> A = zeros(m,n);',\n",
       "  '',\n",
       "  'julia> for i=1:m',\n",
       "  '',\n",
       "  'for j=1:n',\n",
       "  '',\n",
       "  'A[i,j] = 1.0 / norm([pixels[i,:]; 0] - lamps[j,:])^2;',\n",
       "  '',\n",
       "  'end;',\n",
       "  '',\n",
       "  'end;',\n",
       "  '',\n",
       "  'julia> A = (m/sum(A)) * A; # scale elements of A',\n",
       "  '',\n",
       "  'julia> # Least squares solution',\n",
       "  '',\n",
       "  'julia> x = A \\\\ ones(m,1);',\n",
       "  '',\n",
       "  'julia> rms_ls = rms(A*x .- 1)',\n",
       "  '',\n",
       "  '0.1403904813427606',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> histogram(A*x, bins = (0.375:0.05:1.625),',\n",
       "  '',\n",
       "  'legend = false, ylim = (0,120))',\n",
       "  '',\n",
       "  'julia> # Intensity if all lamp powers are one',\n",
       "  '',\n",
       "  'julia> rms_uniform = rms(A*ones(n,1) .- 1)',\n",
       "  '',\n",
       "  '0.24174131853807881',\n",
       "  '',\n",
       "  'julia> histogram(A*ones(n,1), bins = (0.375:0.05:1.625),',\n",
       "  '',\n",
       "  'legend = false, ylim = (0,120))',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '98 12 Least squares',\n",
       "  '',\n",
       "  '0.50 0.75 1.00 1.25 1.50',\n",
       "  '0',\n",
       "  '',\n",
       "  '20',\n",
       "  '',\n",
       "  '40',\n",
       "  '',\n",
       "  '60',\n",
       "  '',\n",
       "  '80',\n",
       "  '',\n",
       "  '100',\n",
       "  '',\n",
       "  '120',\n",
       "  '',\n",
       "  '0.50 0.75 1.00 1.25 1.50',\n",
       "  '0',\n",
       "  '',\n",
       "  '20',\n",
       "  '',\n",
       "  '40',\n",
       "  '',\n",
       "  '60',\n",
       "  '',\n",
       "  '80',\n",
       "  '',\n",
       "  '100',\n",
       "  '',\n",
       "  '120',\n",
       "  '',\n",
       "  'Figure 12.1 Histogram of pixel illumination values using p = 1 (top) and p̂',\n",
       "  '(bottom). The target intensity value is one.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Chapter 13',\n",
       "  '',\n",
       "  'Least squares data fitting',\n",
       "  '',\n",
       "  '13.1 Least squares data fitting',\n",
       "  '',\n",
       "  'Straight-line fit. A straight-line fit to time series data gives an estimate of a',\n",
       "  'trend line. In Figure 13.3 of VMLS we apply this to a time series of petroleum',\n",
       "  'consumption. The figure is reproduced here as Figure 13.1.',\n",
       "  '',\n",
       "  'julia> # Petroleum consumption in thousand barrels/day',\n",
       "  '',\n",
       "  'julia> consumption = petroleum_consumption_data()',\n",
       "  '',\n",
       "  'julia> n = length(consumption);',\n",
       "  '',\n",
       "  'julia> A = [ ones(n) 1:n ];',\n",
       "  '',\n",
       "  'julia> x = A \\\\ consumption;',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> scatter(1980:2013, consumption, legend=false)',\n",
       "  '',\n",
       "  'julia> plot!(1980:2013, A*x)',\n",
       "  '',\n",
       "  'Estimation of trend and seasonal component. The next example is the least',\n",
       "  'squares fit of a trend plus a periodic component to a time series. In VMLS this',\n",
       "  'was illustrated with a time series of vehicle miles traveled in the US, per month,',\n",
       "  'for 15 years (2000–2014). The following Julia code replicates Figure 13.5 in VMLS.',\n",
       "  'It imports the data via the function vehicle_miles_data, which creates a 15× 12',\n",
       "  'matrix vmt, with the monthly values for each of the 15 years.',\n",
       "  '',\n",
       "  'julia> vmt = vehicle_miles_data(); # creates 15x12 matrix vmt',\n",
       "  '',\n",
       "  'julia> m = 15*12;',\n",
       "  '',\n",
       "  'julia> A = [ 0:(m-1) vcat([eye(12) for i=1:15]...) ];',\n",
       "  '',\n",
       "  \"julia> b = reshape(vmt', m, 1);\",\n",
       "  'julia> x = A \\\\ b;',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.13.3',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.13.5',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '100 13 Least squares data fitting',\n",
       "  '',\n",
       "  '1980 1990 2000 2010',\n",
       "  '',\n",
       "  '6×104',\n",
       "  '',\n",
       "  '7×104',\n",
       "  '',\n",
       "  '8×104',\n",
       "  '',\n",
       "  '9×104',\n",
       "  '',\n",
       "  'Figure 13.1 World petroleum consumption between 1980 and 2013 (dots)',\n",
       "  'and least squares straight-line fit (data from www.eia.gov).',\n",
       "  '',\n",
       "  'julia> scatter(1:m, b, markersize = 2, legend =false);',\n",
       "  '',\n",
       "  'julia> plot!(1:m, A*x)',\n",
       "  '',\n",
       "  'The matrix A in this example has size m×n where m = 15 · 12 = 180 and n = 13.',\n",
       "  'The first column has entries 0, 1, 2, . . . , 179. The remaining columns are formed',\n",
       "  'by vertical stacking of 15 identity matrices of size 12 × 12. The Julia expression',\n",
       "  'vcat([eye(12) for i=1:15]...) creates an array of 15 identity matrices, and',\n",
       "  'then stacks them vertically. The plot produced by the code is shown in Figure 13.2.',\n",
       "  '',\n",
       "  'Polynomial fit. We now discuss the polynomial fitting problem on page 255 in',\n",
       "  'VMLS and the results shown in Figure 13.6. We first generate a training set of 100',\n",
       "  'points and plot them (Figure 13.3).',\n",
       "  '',\n",
       "  'julia> # Generate training data in the interval [-1, 1].',\n",
       "  '',\n",
       "  'julia> m = 100;',\n",
       "  '',\n",
       "  'julia> t = -1 .+ 2*rand(m,1);',\n",
       "  '',\n",
       "  'julia> y = t.^3 - t + 0.4 ./ (1 .+ 25*t.^2) + 0.10*randn(m,1);',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> scatter(t,y,legend=false)',\n",
       "  '',\n",
       "  'Next we define a function that fits the polynomial coefficients using least squares.',\n",
       "  'We apply the function to fit polynomials of degree 2, 6, 10, 15 to our training set.',\n",
       "  '',\n",
       "  'julia> polyfit(t, y, p) = vandermonde(t, p) \\\\ y',\n",
       "  '',\n",
       "  'julia> theta2 = polyfit(t,y,3)',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.300',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.13.6',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '13.1 Least squares data fitting 101',\n",
       "  '',\n",
       "  '0 50 100 150',\n",
       "  '',\n",
       "  '2.0×105',\n",
       "  '',\n",
       "  '2.2×105',\n",
       "  '',\n",
       "  '2.4×105',\n",
       "  '',\n",
       "  '2.6×105',\n",
       "  '',\n",
       "  'Figure 13.2 The dots show vehicle miles traveled in the US, per month, in',\n",
       "  'the period January 2000–December 2014. The line shows the least squares',\n",
       "  'fit of a linear trend and a seasonal component with a 12-month period.',\n",
       "  '',\n",
       "  '-1.0 -0.5 0.0 0.5 1.0',\n",
       "  '',\n",
       "  '-0.50',\n",
       "  '',\n",
       "  '-0.25',\n",
       "  '',\n",
       "  '0.00',\n",
       "  '',\n",
       "  '0.25',\n",
       "  '',\n",
       "  '0.50',\n",
       "  '',\n",
       "  'Figure 13.3 Training set used in the polynomial fitting example.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '102 13 Least squares data fitting',\n",
       "  '',\n",
       "  'julia> theta6 = polyfit(t,y,7)',\n",
       "  '',\n",
       "  'julia> theta10 = polyfit(t,y,11)',\n",
       "  '',\n",
       "  'julia> theta15 = polyfit(t,y,16)',\n",
       "  '',\n",
       "  'Finally, we plot the four polynomials. To simplify this, we first write a function',\n",
       "  'that evaluates a polynomial at all points specified in a vector x. The plots are in',\n",
       "  'Figure 13.4.',\n",
       "  '',\n",
       "  'julia> polyeval(theta, x) = vandermonde(x,length(theta))*theta;',\n",
       "  '',\n",
       "  'julia> t_plot = linspace(-1,1,1000);',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> p = plot(layout=4, legend=false, ylim=(-0.7. 0.7))',\n",
       "  '',\n",
       "  'julia> scatter!(t, y, subplot=1, markersize = 2)',\n",
       "  '',\n",
       "  'julia> plot!(t_plot, polyeval(theta2,t_plot), subplot=1)',\n",
       "  '',\n",
       "  'julia> scatter!(t, y, subplot=2, markersize = 2)',\n",
       "  '',\n",
       "  'julia> plot!(t_plot, polyeval(theta6,t_plot), subplot=2)',\n",
       "  '',\n",
       "  'julia> scatter!(t, y, subplot=3,markersize = 2)',\n",
       "  '',\n",
       "  'julia> plot!(t_plot, polyeval(theta10,t_plot), subplot=3)',\n",
       "  '',\n",
       "  'julia> scatter!(t, y, subplot=4, markersize = 2)',\n",
       "  '',\n",
       "  'julia> plot!(t_plot, polyeval(theta15,t_plot), subplot=4)',\n",
       "  '',\n",
       "  'Piecewise-linear fit. In the following code least squares is used to fit a piecewise-',\n",
       "  'linear function to 100 points. It produces Figure 13.5, which is similar to Figure 13.8',\n",
       "  'in VMLS.',\n",
       "  '',\n",
       "  'julia> # generate random data',\n",
       "  '',\n",
       "  'julia> m = 100;',\n",
       "  '',\n",
       "  'julia> x = -2 .+ 4*rand(m,1);',\n",
       "  '',\n",
       "  'julia> y = 1 .+ 2*(x.-1) - 3*max.(x.+1,0) + 4*max.(x.-1,0)',\n",
       "  '',\n",
       "  '+ 0.3*randn(m,1);',\n",
       "  '',\n",
       "  'julia> # least squares fitting',\n",
       "  '',\n",
       "  'julia> theta = [ ones(m) x max.(x.+1,0) max.(x.-1,0) ] \\\\ y;',\n",
       "  '',\n",
       "  'julia> # plot result',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> t = [-2.1, -1, 1, 2.1];',\n",
       "  '',\n",
       "  'julia> yhat = theta[1] .+ theta[2]*t + theta[3]*max.(t.+1,0) +',\n",
       "  '',\n",
       "  'theta[4]*max.(t.-1,0);',\n",
       "  '',\n",
       "  'julia> scatter(x, y, legend=false)',\n",
       "  '',\n",
       "  'julia> plot!(t, yhat)',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.13.8',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '13.1 Least squares data fitting 103',\n",
       "  '',\n",
       "  '-1.0 -0.5 0.0 0.5 1.0',\n",
       "  '',\n",
       "  '-0.50',\n",
       "  '',\n",
       "  '-0.25',\n",
       "  '',\n",
       "  '0.00',\n",
       "  '',\n",
       "  '0.25',\n",
       "  '',\n",
       "  '0.50',\n",
       "  '',\n",
       "  '-1.0 -0.5 0.0 0.5 1.0',\n",
       "  '',\n",
       "  '-0.50',\n",
       "  '',\n",
       "  '-0.25',\n",
       "  '',\n",
       "  '0.00',\n",
       "  '',\n",
       "  '0.25',\n",
       "  '',\n",
       "  '0.50',\n",
       "  '',\n",
       "  '-1.0 -0.5 0.0 0.5 1.0',\n",
       "  '',\n",
       "  '-0.50',\n",
       "  '',\n",
       "  '-0.25',\n",
       "  '',\n",
       "  '0.00',\n",
       "  '',\n",
       "  '0.25',\n",
       "  '',\n",
       "  '0.50',\n",
       "  '',\n",
       "  '-1.0 -0.5 0.0 0.5 1.0',\n",
       "  '',\n",
       "  '-0.50',\n",
       "  '',\n",
       "  '-0.25',\n",
       "  '',\n",
       "  '0.00',\n",
       "  '',\n",
       "  '0.25',\n",
       "  '',\n",
       "  '0.50',\n",
       "  '',\n",
       "  'Figure 13.4 Least squares polynomial fits of degree 1, 6, 10, and 15 to 100',\n",
       "  'points',\n",
       "  '',\n",
       "  '-2 -1 0 1 2',\n",
       "  '',\n",
       "  '-5',\n",
       "  '',\n",
       "  '-4',\n",
       "  '',\n",
       "  '-3',\n",
       "  '',\n",
       "  '-2',\n",
       "  '',\n",
       "  ';',\n",
       "  '',\n",
       "  'Figure 13.5 Piecewise-linear fit to 100 points.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '104 13 Least squares data fitting',\n",
       "  '',\n",
       "  'House price regression. We calculate the simple regression model for predicting',\n",
       "  'house sales price from area and number of bedrooms, using the data of 774 house',\n",
       "  'sales in Sacramento.',\n",
       "  '',\n",
       "  'julia> D = house_sales_data(); # creates 3 vectors: area, beds, price',\n",
       "  '',\n",
       "  'julia> area = D[\"area\"];',\n",
       "  '',\n",
       "  'julia> beds = D[\"beds\"];',\n",
       "  '',\n",
       "  'julia> price = D[\"price\"];',\n",
       "  '',\n",
       "  'julia> m = length(price);',\n",
       "  '',\n",
       "  'julia> A = [ ones(m) area beds ];',\n",
       "  '',\n",
       "  'julia> x = A \\\\ price',\n",
       "  '',\n",
       "  '3-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '54.4017',\n",
       "  '',\n",
       "  '148.725',\n",
       "  '',\n",
       "  '-18.8534',\n",
       "  '',\n",
       "  'julia> rms_error = rms(price - A*x)',\n",
       "  '',\n",
       "  '74.84571649590146',\n",
       "  '',\n",
       "  'julia> std_prices = stdev(price)',\n",
       "  '',\n",
       "  '112.7821615975651',\n",
       "  '',\n",
       "  'Auto-regressive time series model. In the following Julia code we fit an auto-',\n",
       "  'regressive model to the temperature time series discussed on page 259 of VMLS. In',\n",
       "  'Figure 13.6 we compare the first five days of the model predictions with the data.',\n",
       "  '',\n",
       "  'julia> # import time series of temperatures t',\n",
       "  '',\n",
       "  'julia> t = temperature_data();',\n",
       "  '',\n",
       "  'julia> N = length(t)',\n",
       "  '',\n",
       "  '744',\n",
       "  '',\n",
       "  'julia> stdev(t) # Standard deviation',\n",
       "  '',\n",
       "  '3.05055928562933',\n",
       "  '',\n",
       "  'julia> # RMS error for simple predictor zhat_{t+1} = z_t',\n",
       "  '',\n",
       "  'rms(t[2:end] - t[1:end-1])',\n",
       "  '',\n",
       "  '1.1602431638206119',\n",
       "  '',\n",
       "  'julia> # RMS error for simple predictor zhat_{t+1} = z_{t-23}',\n",
       "  '',\n",
       "  'rms(t[25:end] - t[1:end-24])',\n",
       "  '',\n",
       "  '1.7338941400468744',\n",
       "  '',\n",
       "  'julia> # Least squares fit of AR predictor with memory 8',\n",
       "  '',\n",
       "  'julia> M = 8',\n",
       "  '',\n",
       "  'julia> y = t[M+1:end];',\n",
       "  '',\n",
       "  'julia> A = hcat( [ t[i:i+N-M-1] for i = M:-1:1]...);',\n",
       "  '',\n",
       "  'julia> theta = A \\\\ y;',\n",
       "  '',\n",
       "  'julia> ypred = A*theta;',\n",
       "  '',\n",
       "  'julia> # RMS error of LS AR fit',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.304',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '13.2 Validation 105',\n",
       "  '',\n",
       "  '0 25 50 75 100',\n",
       "  '',\n",
       "  '57.5',\n",
       "  '',\n",
       "  '60.0',\n",
       "  '',\n",
       "  '62.5',\n",
       "  '',\n",
       "  '65.0',\n",
       "  '',\n",
       "  '67.5',\n",
       "  '',\n",
       "  'Figure 13.6 Hourly temperature at Los Angeles International Airport be-',\n",
       "  'tween 12:53AM on May 1, 2016, and 11:53PM on May 5, 2016, shown as',\n",
       "  'circles. The solid line is the prediction of an auto-regressive model with eight',\n",
       "  'coefficients.',\n",
       "  '',\n",
       "  'julia> rms(ypred - y)',\n",
       "  '',\n",
       "  '1.0129632612687514',\n",
       "  '',\n",
       "  'julia> # Plot first five days',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> Nplot = 24*5',\n",
       "  '',\n",
       "  'julia> scatter(1:Nplot, t[1:Nplot], legend =false)',\n",
       "  '',\n",
       "  'julia> plot!(M+1:Nplot, ypred[1:Nplot-M])',\n",
       "  '',\n",
       "  '13.2 Validation',\n",
       "  '',\n",
       "  'Polynomial approximation. We return to the polynomial fitting example of page 100.',\n",
       "  'We continue with the data vectors t and y in the code on page 100 as the training',\n",
       "  'set, and generate a test set of 100 randomly chosen points generated by the same',\n",
       "  'method as used for the training set. We then fit polynomials of degree 0,. . . , 20',\n",
       "  '(i.e., with p = 1, . . . , 21 coefficients) and compute the RMS errors on the training',\n",
       "  'set and the test set. This produces a figure similar to Figure 13.11 in VMLS, shown',\n",
       "  'here as Figure 13.7.',\n",
       "  '',\n",
       "  'julia> # Generate the test set.',\n",
       "  '',\n",
       "  'julia> m = 100;',\n",
       "  '',\n",
       "  'julia> t_test = -1 .+ 2*rand(m,1);',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.13.11',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '106 13 Least squares data fitting',\n",
       "  '',\n",
       "  '0 5 10 15 20',\n",
       "  '',\n",
       "  '0.3',\n",
       "  '',\n",
       "  '0.4',\n",
       "  '',\n",
       "  '0.5',\n",
       "  '',\n",
       "  '0.6',\n",
       "  '',\n",
       "  '0.7',\n",
       "  '',\n",
       "  '0.8',\n",
       "  '',\n",
       "  '0.9',\n",
       "  '',\n",
       "  'Degree',\n",
       "  '',\n",
       "  'R',\n",
       "  'el',\n",
       "  '',\n",
       "  'at',\n",
       "  'iv',\n",
       "  '',\n",
       "  'e ',\n",
       "  'R',\n",
       "  '',\n",
       "  'M',\n",
       "  'S',\n",
       "  '',\n",
       "  ' e',\n",
       "  'rr',\n",
       "  '',\n",
       "  'or',\n",
       "  '',\n",
       "  'Train',\n",
       "  'Test',\n",
       "  '',\n",
       "  ';',\n",
       "  '',\n",
       "  'Figure 13.7 RMS error versus polynomial degree for the fitting example in',\n",
       "  'figure 13.4.',\n",
       "  '',\n",
       "  'julia> y_test = t_test.^3 - t_test + 0.4 ./ (1 .+ 25*t_test.^2)',\n",
       "  '',\n",
       "  '+ 0.10*randn(m,1);',\n",
       "  '',\n",
       "  'julia> error_train = zeros(21);',\n",
       "  '',\n",
       "  'julia> error_test = zeros(21);',\n",
       "  '',\n",
       "  'julia> for p = 1:21',\n",
       "  '',\n",
       "  'A = vandermonde(t,p)',\n",
       "  '',\n",
       "  'theta = A \\\\ y',\n",
       "  '',\n",
       "  'error_train[p] = norm(A*theta - y) / norm(y)',\n",
       "  '',\n",
       "  'error_test[p] = norm( vandermonde(t_test, p) * theta',\n",
       "  '',\n",
       "  '- y_test) / norm(y_test);',\n",
       "  '',\n",
       "  'end',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> plot(0:20, error_train, label = \"Train\", marker = :circle)',\n",
       "  '',\n",
       "  'julia> plot!(0:20, error_test, label = \"Test\", marker = :square)',\n",
       "  '',\n",
       "  'julia> plot!(xlabel=\"Degree\", ylabel = \"Relative RMS error\")',\n",
       "  '',\n",
       "  'House price regression model. On page 13.1 we used a data set of 774 house',\n",
       "  'sales data to fit a simple regression model',\n",
       "  '',\n",
       "  'ŷ = v + β1x1 + β2x2,',\n",
       "  '',\n",
       "  'where ŷ is the predicted sales price, x1 is the area, and x2 is the number of bed-',\n",
       "  'rooms. Here we apply cross-validation to assess the generalization ability of the',\n",
       "  'simple model. We use five folds, four of size 155 (Nfold in the code below) and one',\n",
       "  'of size 154. To choose the five folds, we create a random permutation of the indices',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '13.2 Validation 107',\n",
       "  '',\n",
       "  '1, . . . , 774. (We do this by calling the randperm function in the Random package.)',\n",
       "  'We choose the data points indexed by the first 155 elements in the permuted list as',\n",
       "  'fold 1, the next 155 as fold 2, et cetera. The output of the following code outputs',\n",
       "  'is similar to Table 13.1 in VMLS (with different numbers because of the random',\n",
       "  'choice of folds).',\n",
       "  '',\n",
       "  'julia> D = house_sales_data();',\n",
       "  '',\n",
       "  'julia> price = D[\"price\"]; area = D[\"area\"]; beds = D[\"beds\"];',\n",
       "  '',\n",
       "  'julia> N = length(price);',\n",
       "  '',\n",
       "  'julia> X = [ ones(N) area beds ];',\n",
       "  '',\n",
       "  'julia> nfold = div(N,5); # size of first four folds',\n",
       "  '',\n",
       "  'julia> import Random',\n",
       "  '',\n",
       "  'julia> I = Random.randperm(N); # random permutation of numbers 1...N',\n",
       "  '',\n",
       "  'julia> coeff = zeros(5,3); errors = zeros(5,2);',\n",
       "  '',\n",
       "  'julia> for k = 1:5',\n",
       "  '',\n",
       "  'if k == 1',\n",
       "  '',\n",
       "  'Itrain = I[nfold+1:end];',\n",
       "  '',\n",
       "  'Itest = I[1:nfold];',\n",
       "  '',\n",
       "  'elseif k == 5',\n",
       "  '',\n",
       "  'Itrain = I[1:4*nfold];',\n",
       "  '',\n",
       "  'Itest = I[4*nfold+1:end];',\n",
       "  '',\n",
       "  'else',\n",
       "  '',\n",
       "  'Itrain = I[ [1:(k-1)*nfold ; k*nfold+1 : N]]',\n",
       "  '',\n",
       "  'Itest = I[ [(k-1)*nfold+1 ; k*nfold ]];',\n",
       "  '',\n",
       "  'end;',\n",
       "  '',\n",
       "  'Ntrain = length(Itrain)',\n",
       "  '',\n",
       "  'Ntest = length(Itest)',\n",
       "  '',\n",
       "  'theta = X[Itrain,:] \\\\ price[Itrain];',\n",
       "  '',\n",
       "  'coeff[k,:] = theta;',\n",
       "  '',\n",
       "  'rms_train = rms(X[Itrain,:] * theta - price[Itrain])',\n",
       "  '',\n",
       "  'rms_test = rms(X[Itest,:] * theta - price[Itest])',\n",
       "  '',\n",
       "  'end;',\n",
       "  '',\n",
       "  'julia> coeff # 3 coefficients for the five folds',\n",
       "  '',\n",
       "  '5×3 Array{Float64,2}:',\n",
       "  '56.4566 150.822 -20.1493',\n",
       "  '',\n",
       "  '59.4955 148.725 -20.0837',\n",
       "  '',\n",
       "  '53.7978 154.69 -21.577',\n",
       "  '',\n",
       "  '56.1279 145.629 -18.2875',\n",
       "  '',\n",
       "  '46.2149 144.207 -14.4336',\n",
       "  '',\n",
       "  'julia> [rms_train rms_test] # RMS errors for five folds',\n",
       "  '',\n",
       "  '5×2 Array{Float64,2}:',\n",
       "  '76.0359 69.9312',\n",
       "  '',\n",
       "  '75.4704 74.0014',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#table.13.1',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '108 13 Least squares data fitting',\n",
       "  '',\n",
       "  '76.194 94.3307',\n",
       "  '',\n",
       "  '73.3722 27.6299',\n",
       "  '',\n",
       "  '72.9218 82.1444',\n",
       "  '',\n",
       "  'Validating time series predictions. In the next example, we return to the AR',\n",
       "  'model of hourly temperatures at LAX. We divide the time series in a training set',\n",
       "  'of 24 days and a test set of 7 days. We fit the AR model to the training set and',\n",
       "  'calculate the RMS prediction errors on the training and test sets. Figure 13.8',\n",
       "  'shows the model predictions and the the data for the first five days of the test set.',\n",
       "  '',\n",
       "  'julia> t = temperature_data();',\n",
       "  '',\n",
       "  'julia> N = length(t);',\n",
       "  '',\n",
       "  'julia> # use first 24 days as training set',\n",
       "  '',\n",
       "  'julia> Ntrain = 24 * 24; t_train = t[1:Ntrain];',\n",
       "  '',\n",
       "  'julia> # use the rest as test set',\n",
       "  '',\n",
       "  'julia> Ntest = N-Ntrain; t_test = t[Ntrain+1:end];',\n",
       "  '',\n",
       "  'julia> # Least squares fit of AR predictor with memory 8',\n",
       "  '',\n",
       "  'julia> m = Ntrain - M;',\n",
       "  '',\n",
       "  'julia> y = t_train[M+1:M+m];',\n",
       "  '',\n",
       "  'julia> A = hcat( [t_train[i:i+m-1] for i=M:-1:1]...);',\n",
       "  '',\n",
       "  'julia> coeff = A \\\\ y;',\n",
       "  '',\n",
       "  'julia> rms_train = rms(A*coeff-y)',\n",
       "  '',\n",
       "  '1.0253577259862334',\n",
       "  '',\n",
       "  'julia> ytest = t_test[M+1:end];',\n",
       "  '',\n",
       "  'julia> mtest = length(ytest);',\n",
       "  '',\n",
       "  'julia> ypred = hcat( [t_test[i:i+mtest-1] for i=M:-1:1]...) * coeff;',\n",
       "  '',\n",
       "  'julia> rms_test = rms(ypred - ytest)',\n",
       "  '',\n",
       "  '0.9755113632200967',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> Nplot = 24*5',\n",
       "  '',\n",
       "  'julia> scatter(1:Nplot, t_test[1:Nplot], legend=false)',\n",
       "  '',\n",
       "  'julia> plot!(M+1:Nplot, ypred[1:Nplot-M])',\n",
       "  '',\n",
       "  '13.3 Feature engineering',\n",
       "  '',\n",
       "  'Next we compute the more complicated house price regression model of §13.3.5 of',\n",
       "  'VMLS. The data are imported via the function house_sales_data, which returns',\n",
       "  'a dictionary containing the following five vectors of length 774:',\n",
       "  '',\n",
       "  '• price: price in thousand dollars,',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#subsection.13.3.5',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '13.3 Feature engineering 109',\n",
       "  '',\n",
       "  '0 25 50 75 100',\n",
       "  '',\n",
       "  '57.5',\n",
       "  '',\n",
       "  '60.0',\n",
       "  '',\n",
       "  '62.5',\n",
       "  '',\n",
       "  '65.0',\n",
       "  '',\n",
       "  '67.5',\n",
       "  '',\n",
       "  'Figure 13.8 Hourly temperature at Los Angeles International Airport be-',\n",
       "  'tween 12:53AM on May 25, 2016, and 11:53PM on May 29, 2016, shown',\n",
       "  'as circles. The solid line is the prediction of an auto-regressive model with',\n",
       "  'eight coefficients, developed using training data from May 1 to May 24.',\n",
       "  '',\n",
       "  '• area: house area in 1000 square feet,',\n",
       "  '',\n",
       "  '• beds: number of bedrooms,',\n",
       "  '',\n",
       "  '• condo: 1 if a condo, 0 otherwise,',\n",
       "  '',\n",
       "  '• location: a number from 1–4, for the four sets of ZIP codes in Table 13.4',\n",
       "  'of VMLS.',\n",
       "  '',\n",
       "  'The code computes the model and makes a scatter plot of actual and predicted',\n",
       "  'prices (Figure 13.9). Note that the last three columns of the matrix X contain',\n",
       "  'Boolean variables (true or false). We rely on the fact that Julia treats this as',\n",
       "  'integers 1 and 0.',\n",
       "  '',\n",
       "  'julia> D = house_sales_data();',\n",
       "  '',\n",
       "  'julia> price = D[\"price\"];',\n",
       "  '',\n",
       "  'julia> area = D[\"area\"];',\n",
       "  '',\n",
       "  'julia> beds = D[\"beds\"];',\n",
       "  '',\n",
       "  'julia> condo = D[\"condo\"];',\n",
       "  '',\n",
       "  'julia> location = D[\"location\"];',\n",
       "  '',\n",
       "  'julia> N = length(price);',\n",
       "  '',\n",
       "  'julia> X = hcat(ones(N), area, max.(area.-1.5, 0), beds, condo,',\n",
       "  '',\n",
       "  'location .== 2, location .== 3, location .== 4 );',\n",
       "  '',\n",
       "  'julia> theta = X \\\\ price',\n",
       "  '',\n",
       "  '8-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '115.617',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#table.13.4',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '110 13 Least squares data fitting',\n",
       "  '',\n",
       "  '0 200 400 600 800',\n",
       "  '0',\n",
       "  '',\n",
       "  '200',\n",
       "  '',\n",
       "  '400',\n",
       "  '',\n",
       "  '600',\n",
       "  '',\n",
       "  '800',\n",
       "  '',\n",
       "  'Actual price',\n",
       "  '',\n",
       "  'P',\n",
       "  're',\n",
       "  '',\n",
       "  'di',\n",
       "  'ct',\n",
       "  '',\n",
       "  'ed',\n",
       "  ' p',\n",
       "  '',\n",
       "  'ric',\n",
       "  'e',\n",
       "  '',\n",
       "  'Figure 13.9 Scatter plot of actual and predicted prices for a model with eight',\n",
       "  'parameters.',\n",
       "  '',\n",
       "  '175.413',\n",
       "  '',\n",
       "  '-42.7478',\n",
       "  '',\n",
       "  '-17.8784',\n",
       "  '',\n",
       "  '-19.0447',\n",
       "  '',\n",
       "  '-100.911',\n",
       "  '',\n",
       "  '-108.791',\n",
       "  '',\n",
       "  '-24.7652',\n",
       "  '',\n",
       "  'julia> rms(X*theta - price) # RMS prediction error',\n",
       "  '',\n",
       "  '68.34428699036884',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> scatter(price, X*theta, lims = (0,800));',\n",
       "  '',\n",
       "  'julia> plot!([0, 800], [0, 800], linestyle = :dash, legend = false);',\n",
       "  '',\n",
       "  'julia> plot!(xlims = (0,800), ylims = (0,800), size = (500,500));',\n",
       "  '',\n",
       "  'julia> plot!(xlabel = \"Actual price\", ylabel = \"Predicted price\");',\n",
       "  '',\n",
       "  'We finish by a cross-validation of this method. We follow the same approach as for',\n",
       "  'the simple regression model on page 106, using five randomly chosen folds. The',\n",
       "  'code shows the eight coefficients, and the RMS training and test errors for each',\n",
       "  'fold.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '13.3 Feature engineering 111',\n",
       "  '',\n",
       "  'julia> nfold = div(N,5);',\n",
       "  '',\n",
       "  'julia> import Random;',\n",
       "  '',\n",
       "  'julia> I = Random.randperm(N);',\n",
       "  '',\n",
       "  'julia> models = zeros(8,5); # store 8 coefficients for the 5 models',\n",
       "  '',\n",
       "  'julia> errors = zeros(2,5); # prediction errors',\n",
       "  '',\n",
       "  'julia> for k = 1:5',\n",
       "  '',\n",
       "  'if k == 1',\n",
       "  '',\n",
       "  'Itrain = I[nfold+1:end];',\n",
       "  '',\n",
       "  'Itest = I[1:nfold];',\n",
       "  '',\n",
       "  'elseif k == 5',\n",
       "  '',\n",
       "  'Itrain = I[1:4*nfold];',\n",
       "  '',\n",
       "  'Itest = I[4*nfold+1:end];',\n",
       "  '',\n",
       "  'else',\n",
       "  '',\n",
       "  'Itrain = I[ [1:(k-1)*nfold ; k*nfold+1 : N]]',\n",
       "  '',\n",
       "  'Itest = I[ [(k-1)*nfold+1 ; k*nfold ]];',\n",
       "  '',\n",
       "  'end;',\n",
       "  '',\n",
       "  'Ntrain = length(Itrain)',\n",
       "  '',\n",
       "  'Ntest = length(Itest)',\n",
       "  '',\n",
       "  'theta = X[Itrain,:] \\\\ price[Itrain];',\n",
       "  '',\n",
       "  'errors[1,k] = rms(X[Itrain,:] * theta - price[Itrain]);',\n",
       "  '',\n",
       "  'errors[2,k] = rms(X[Itest,:] * theta - price[Itest]);',\n",
       "  '',\n",
       "  'models[:,k] = theta;',\n",
       "  '',\n",
       "  'end;',\n",
       "  '',\n",
       "  'julia> # display the eigth coefficients for each of the 5 folds',\n",
       "  '',\n",
       "  'julia> models',\n",
       "  '',\n",
       "  '8×5 Array{Float64,2}:',\n",
       "  '121.294 110.602 142.51 103.589 101.677',\n",
       "  '',\n",
       "  '170.343 167.999 165.442 187.179 184.096',\n",
       "  '',\n",
       "  '-32.3645 -39.6754 -35.3901 -51.6821 -52.7473',\n",
       "  '',\n",
       "  '-21.0324 -16.5711 -15.0365 -17.972 -17.9788',\n",
       "  '',\n",
       "  '-17.6111 -27.6004 -19.4401 -13.7243 -16.2317',\n",
       "  '',\n",
       "  '-92.5554 -87.638 -124.744 -103.573 -97.4796',\n",
       "  '',\n",
       "  '-98.3588 -97.6417 -131.886 -111.991 -105.518',\n",
       "  '',\n",
       "  '-7.77581 -13.3224 -58.3521 -33.7546 -13.7175',\n",
       "  '',\n",
       "  'julia> # display training errors (1st row) and test errors (2nd row)',\n",
       "  '',\n",
       "  'julia> errors',\n",
       "  '',\n",
       "  '2×5 Array{Float64,2}:',\n",
       "  '66.1225 69.3897 66.0522 70.2625 69.16',\n",
       "  '',\n",
       "  '77.193 42.2776 115.636 73.5962 65.4291',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '112 13 Least squares data fitting',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Chapter 14',\n",
       "  '',\n",
       "  'Least squares classification',\n",
       "  '',\n",
       "  '14.1 Classification',\n",
       "  '',\n",
       "  'Boolean values. Julia has the Boolean values true and false. These are au-',\n",
       "  'tomatically converted to the numbers 1 and 0 when they combined in numerical',\n",
       "  'expressions. In VMLS we use the encoding (for classifiers) where True corresponds',\n",
       "  'to +1 and False corresponds to −1. We can get our encoding from a Julia Boolean',\n",
       "  'value b using 2*b-1, or via the ternary conditional operation b ? 1 : -1.',\n",
       "  '',\n",
       "  'julia> tf2pm1(b) = 2*b-1',\n",
       "  '',\n",
       "  'julia> b = true',\n",
       "  '',\n",
       "  'true',\n",
       "  '',\n",
       "  'julia> tf2pm1(b)',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  'julia> b = false',\n",
       "  '',\n",
       "  'false',\n",
       "  '',\n",
       "  'julia> tf2pm1(b)',\n",
       "  '',\n",
       "  '-1',\n",
       "  '',\n",
       "  'julia> b = [ true, false, true ]',\n",
       "  '',\n",
       "  '3-element Array{Bool,1}:',\n",
       "  '',\n",
       "  'true',\n",
       "  '',\n",
       "  'false',\n",
       "  '',\n",
       "  'true',\n",
       "  '',\n",
       "  'julia> tf2pm1.(b)',\n",
       "  '',\n",
       "  '3-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '-1',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '114 14 Least squares classification',\n",
       "  '',\n",
       "  'Confusion matrix. Let’s see how we would evaluate the prediction errors and',\n",
       "  'confusion matrix, given a set of data y and predictions yhat, both stored as arrays',\n",
       "  '(vectors) of Boolean values, of length N.',\n",
       "  '',\n",
       "  'julia> # Count errors and correct predictions',\n",
       "  '',\n",
       "  'julia> Ntp(y,yhat) = sum( (y .== true) .& (yhat .== true) );',\n",
       "  '',\n",
       "  'julia> Nfn(y,yhat) = sum( (y .== true) .& (yhat .== false) );',\n",
       "  '',\n",
       "  'julia> Nfp(y,yhat) = sum( (y .== false) .& (yhat .== true) );',\n",
       "  '',\n",
       "  'julia> Ntn(y,yhat) = sum( (y .== false) .& (yhat .== false) );',\n",
       "  '',\n",
       "  'julia> error_rate(y,yhat) = (Nfn(y,yhat) + Nfp(y,yhat)) / length(y);',\n",
       "  '',\n",
       "  'julia> confusion_matrix(y,yhat) = [ Ntp(y,yhat) Nfn(y,yhat);',\n",
       "  '',\n",
       "  'Nfp(y,yhat) Ntn(y,yhat) ];',\n",
       "  '',\n",
       "  'julia> y = rand(Bool,100); yhat = rand(Bool,100);',\n",
       "  '',\n",
       "  'julia> confusion_matrix(y,yhat)',\n",
       "  '',\n",
       "  '2×2 Array{Int64,2}:',\n",
       "  '25 23',\n",
       "  '',\n",
       "  '29 23',\n",
       "  '',\n",
       "  'julia> error_rate(y,yhat)',\n",
       "  '',\n",
       "  '0.52',\n",
       "  '',\n",
       "  'The dots that precede == and & cause them to be evaluated elementwise. When',\n",
       "  'we sum the Boolean vectors, they are converted to integers. In the last section',\n",
       "  'of the code we generate two random Boolean vectors, so we expect the error',\n",
       "  'rate to be around 50%. In the code above, we compute the error rate from',\n",
       "  'the numbers of false negatives and false positives. A more compact expression',\n",
       "  'for the error rate is avg(y .!= yhat). The VMLS package contains the function',\n",
       "  'confusion_matrix(y, yhat).',\n",
       "  '',\n",
       "  '14.2 Least squares classifier',\n",
       "  '',\n",
       "  'We can evaluate f̂(x) = sign(f̃(x)) using ftilde(x)>0, which returns a Boolean',\n",
       "  'value.',\n",
       "  '',\n",
       "  \"julia> ftilde(x) = x'*beta .+ v # Regression model\",\n",
       "  'julia> fhat(x) = ftilde(x) > 0 # Regression classifier',\n",
       "  '',\n",
       "  'Iris flower classification. The Iris data set contains of 150 examples of three',\n",
       "  'types of iris flowers. There are 50 examples of each class. For each example, four',\n",
       "  'features are provided. The following code reads in a dictionary containing three',\n",
       "  '50× 4 matrices setosa, versicolor, virginica with the examples for each class,',\n",
       "  'and then computes a Boolean classifier that distinguishes Iris Virginica from the',\n",
       "  'the other two classes.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '14.3 Multi-class classifiers 115',\n",
       "  '',\n",
       "  'julia> D = iris_data();',\n",
       "  '',\n",
       "  'julia> # Create 150x4 data matrix',\n",
       "  '',\n",
       "  'julia> iris = vcat(D[\"setosa\"], D[\"versicolor\"], D[\"virginica\"])',\n",
       "  '',\n",
       "  'julia> # y[k] is true (1) if virginica, false (0) otherwise',\n",
       "  '',\n",
       "  'julia> y = [ zeros(Bool, 50); zeros(Bool, 50); ones(Bool, 50) ];',\n",
       "  '',\n",
       "  'julia> A = [ ones(150) iris ]',\n",
       "  '',\n",
       "  'julia> theta = A \\\\ (2*y .- 1)',\n",
       "  '',\n",
       "  '5×1 Array{Float64,2}:',\n",
       "  '-2.39056',\n",
       "  '',\n",
       "  '-0.0917522',\n",
       "  '',\n",
       "  '0.405537',\n",
       "  '',\n",
       "  '0.00797582',\n",
       "  '',\n",
       "  '1.10356',\n",
       "  '',\n",
       "  'julia> yhat = A*theta .> 0;',\n",
       "  '',\n",
       "  'julia> C = confusion_matrix(y,yhat)',\n",
       "  '',\n",
       "  '2×2 Array{Int64,2}:',\n",
       "  '46 4',\n",
       "  '',\n",
       "  '7 93',\n",
       "  '',\n",
       "  'julia> err_rate = (C[1,2] + C[2,1]) / length(y)',\n",
       "  '',\n",
       "  '0.07333333333333333',\n",
       "  '',\n",
       "  'julia> avg(y .!= yhat)',\n",
       "  '',\n",
       "  '0.07333333333333333',\n",
       "  '',\n",
       "  '14.3 Multi-class classifiers',\n",
       "  '',\n",
       "  'Multi-class error rate and confusion matrix. The overall error rate is easily eval-',\n",
       "  'uated as avg(y .!= yhat). We can form the K×K confusion matrix from a set of',\n",
       "  'N true outcomes y and N predictions yhat (each with entries among {1, . . . ,K})',\n",
       "  'by counting the number of times each pair of values occurs.',\n",
       "  '',\n",
       "  'julia> error_rate(y, yhat) = avg(y .!= yhat);',\n",
       "  '',\n",
       "  'julia> function confusion_matrix(y, yhat, K)',\n",
       "  '',\n",
       "  'C = zeros(K,K)',\n",
       "  '',\n",
       "  'for i in 1:K for j in 1:K',\n",
       "  '',\n",
       "  'C[i,j] = sum((y .== i) .& (yhat .== j))',\n",
       "  '',\n",
       "  'end end',\n",
       "  '',\n",
       "  'return C',\n",
       "  '',\n",
       "  'end;',\n",
       "  '',\n",
       "  'julia> # test for K=4 on random vectors of length 100',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '116 14 Least squares classification',\n",
       "  '',\n",
       "  'julia> K = 4;',\n",
       "  '',\n",
       "  'julia> y = rand(1:K, 100); yhat = rand(1:K, 100);',\n",
       "  '',\n",
       "  'julia> C = confusion_matrix(y, yhat, K)',\n",
       "  '',\n",
       "  '4×4 Array{Float64,2}:',\n",
       "  '4.0 9.0 8.0 5.0',\n",
       "  '',\n",
       "  '5.0 4.0 4.0 13.0',\n",
       "  '',\n",
       "  '3.0 8.0 7.0 11.0',\n",
       "  '',\n",
       "  '7.0 2.0 7.0 3.0',\n",
       "  '',\n",
       "  'julia> error_rate(y, yhat), 1-sum(diag(C))/sum(C)',\n",
       "  '',\n",
       "  '(0.82, 0.8200000000000001)',\n",
       "  '',\n",
       "  'The function confusion_matrix is included in the VMLS package.',\n",
       "  '',\n",
       "  'Least squares multi-class classifier. A K-class classifier (with regression model)',\n",
       "  'can be expressed as',\n",
       "  '',\n",
       "  'f̂(x) = argmax',\n",
       "  'k=1,...,K',\n",
       "  '',\n",
       "  'f̃k(x),',\n",
       "  '',\n",
       "  'where f̃k(x) = x',\n",
       "  'T θk. The n-vectors θ1, . . . , θK are the coefficients or parameters',\n",
       "  '',\n",
       "  'in the model. We can express this in matrix-vector notation as',\n",
       "  '',\n",
       "  'f̂(x) = argmax(xTΘ),',\n",
       "  '',\n",
       "  'where Θ = [ θ1 · · · θK ] is the n × K matrix of model coefficients, and the',\n",
       "  'argmax of a row vector has the obvious meaning.',\n",
       "  '',\n",
       "  'Let’s see how to express this in Julia. In Julia the function argmax(u) finds',\n",
       "  'the index of the largest entry in the row or column vector u, i.e., argmaxk uk. To',\n",
       "  'extend this to matrices, we define a function row_argmax that returns a vector',\n",
       "  'with, for each row, the index of the largest entry in that row.',\n",
       "  '',\n",
       "  'julia> row_argmax(u) = [ argmax(u[i,:]) for i = 1:size(u,1) ]',\n",
       "  '',\n",
       "  'julia> A = randn(4,5)',\n",
       "  '',\n",
       "  '4×5 Array{Float64,2}:',\n",
       "  '1.42552 0.766725 1.7106 -1.08668 -0.492051',\n",
       "  '',\n",
       "  '-0.507653 -0.158288 -1.37703 -0.388304 0.00290895',\n",
       "  '',\n",
       "  '-1.43499 -1.18238 -0.182795 -0.428589 -0.87592',\n",
       "  '',\n",
       "  '-2.18407 1.28363 0.749702 -0.304138 0.0165654',\n",
       "  '',\n",
       "  'julia> row_argmax(A)',\n",
       "  '',\n",
       "  '4-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '3',\n",
       "  '',\n",
       "  '5',\n",
       "  '',\n",
       "  '3',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  'If a data set with N examples is stored as an n ×N data matrix X, and Theta is',\n",
       "  'an n ×K matrix with the coefficient vectors θk as its columns, then we can now',\n",
       "  'define a function',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '14.3 Multi-class classifiers 117',\n",
       "  '',\n",
       "  \"fhat(X,Theta) = row_argmax(X'*Theta)\",\n",
       "  '',\n",
       "  'to find the N -vector of predictions.',\n",
       "  '',\n",
       "  'Matrix least squares. Let’s use least squares to find the coefficient matrix Θ for a',\n",
       "  'multi-class classifier with n features and K classes, from a data set of N examples.',\n",
       "  'We will assume the data is given as an n × N matrix X and an N -vector ycl',\n",
       "  'with entries in {1, . . . ,K} that give the classes of the examples. The least squares',\n",
       "  'objective can be expressed as a matrix norm squared,',\n",
       "  '',\n",
       "  '‖XTΘ− Y ‖2,',\n",
       "  '',\n",
       "  'where Y is the N ×K vector with',\n",
       "  '',\n",
       "  'Yij =',\n",
       "  '',\n",
       "  '{',\n",
       "  '1 ycli = j',\n",
       "  '−1 ycli 6= j.',\n",
       "  '',\n",
       "  'In other words, the rows of Y describe the classes using one-hot encoding, converted',\n",
       "  'from 0/1 to −1/+ 1 values. The least squares solution is given by Θ̂ = (XT )†Y .',\n",
       "  '',\n",
       "  'Let’s see how to express this in Julia.',\n",
       "  '',\n",
       "  'julia> function one_hot(ycl,K)',\n",
       "  '',\n",
       "  'N = length(ycl)',\n",
       "  '',\n",
       "  'Y = zeros(N,K)',\n",
       "  '',\n",
       "  'for j in 1:K',\n",
       "  '',\n",
       "  'Y[findall(ycl .== j), j] .= 1',\n",
       "  '',\n",
       "  'end',\n",
       "  '',\n",
       "  'return Y',\n",
       "  '',\n",
       "  'end;',\n",
       "  '',\n",
       "  'julia> K = 4;',\n",
       "  '',\n",
       "  'julia> ycl = rand(1:K,6)',\n",
       "  '',\n",
       "  '6-element Array{Int64,1}:',\n",
       "  '',\n",
       "  '4',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  '3',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  'julia> Y = one_hot(ycl, K)',\n",
       "  '',\n",
       "  '64 ArrayFloat64,2:',\n",
       "  '',\n",
       "  '0.0 0.0 0.0 1.0',\n",
       "  '',\n",
       "  '0.0 1.0 0.0 0.0',\n",
       "  '',\n",
       "  '0.0 0.0 1.0 0.0',\n",
       "  '',\n",
       "  '1.0 0.0 0.0 0.0',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '118 14 Least squares classification',\n",
       "  '',\n",
       "  '0.0 1.0 0.0 0.0',\n",
       "  '',\n",
       "  '1.0 0.0 0.0 0.0',\n",
       "  '',\n",
       "  'julia> 2*Y .- 1',\n",
       "  '',\n",
       "  '6×4 Array{Float64,2}:',\n",
       "  '-1.0 -1.0 -1.0 1.0',\n",
       "  '',\n",
       "  '-1.0 1.0 -1.0 -1.0',\n",
       "  '',\n",
       "  '-1.0 -1.0 1.0 -1.0',\n",
       "  '',\n",
       "  '1.0 -1.0 -1.0 -1.0',\n",
       "  '',\n",
       "  '-1.0 1.0 -1.0 -1.0',\n",
       "  '',\n",
       "  '1.0 -1.0 -1.0 -1.0',\n",
       "  '',\n",
       "  'Using the functions we have defined, the matrix least squares multi-class classifier',\n",
       "  'can be computed in a few lines.',\n",
       "  '',\n",
       "  'julia> function ls_multiclass(X,ycl,K)',\n",
       "  '',\n",
       "  'n, N = size(X)',\n",
       "  '',\n",
       "  \"Theta = X' \\\\ (2*one_hot(ycl,K) .- 1)\",\n",
       "  \"yhat = row_argmax(X'*theta)\",\n",
       "  'return Theta, yhat',\n",
       "  '',\n",
       "  'end',\n",
       "  '',\n",
       "  'Iris flower classification. We compute a 3-class classifier for the iris flower data',\n",
       "  'set. We split the data set of 150 examples in a training set of 120 (40 per class)',\n",
       "  'and a test set of 30 (10 per class). The code calls the functions we defined above.',\n",
       "  '',\n",
       "  'julia> D = iris_data();',\n",
       "  '',\n",
       "  'julia> setosa = D[\"setosa\"];',\n",
       "  '',\n",
       "  'julia> versicolor = D[\"versicolor\"];',\n",
       "  '',\n",
       "  'julia> virginica = D[\"virginica\"];',\n",
       "  '',\n",
       "  'julia> # pick three random permutations of 1,..., 50',\n",
       "  '',\n",
       "  'julia> import Random',\n",
       "  '',\n",
       "  'julia> I1 = Random.randperm(50);',\n",
       "  '',\n",
       "  'julia> I2 = Random.randperm(50);',\n",
       "  '',\n",
       "  'julia> I3 = Random.randperm(50);',\n",
       "  '',\n",
       "  'julia> # training set is 40 randomly picked examples per class',\n",
       "  '',\n",
       "  'julia> Xtrain = [ setosa[I1[1:40],:];',\n",
       "  '',\n",
       "  'versicolor[I2[1:40],:];',\n",
       "  '',\n",
       "  \"virginica[I3[1:40],:] ]'; # 4x120 data matrix\",\n",
       "  'julia> # add constant feature one',\n",
       "  '',\n",
       "  'julia> Xtrain = [ ones(1,120); Xtrain ]; # 5x120 data matrix',\n",
       "  '',\n",
       "  'julia> ytrain = [ ones(40); 2*ones(40); 3*ones(40) ];',\n",
       "  '',\n",
       "  'julia> # test set is remaining 10 examples for each class',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '14.3 Multi-class classifiers 119',\n",
       "  '',\n",
       "  'julia> Xtest = [ setosa[I1[41:end],:];',\n",
       "  '',\n",
       "  'versicolor[I2[41:end],:]',\n",
       "  '',\n",
       "  \"virginica[I3[41:end],:] ]'; # 4x30 data matrix\",\n",
       "  'julia> Xtest = [ ones(1,30); Xtest ]; # 5x30 data matrix',\n",
       "  '',\n",
       "  'julia> ytest = [ones(10); 2*ones(10); 3*ones(10)];',\n",
       "  '',\n",
       "  'julia> Theta, yhat = ls_multiclass(Xtrain, ytrain, 3);',\n",
       "  '',\n",
       "  'julia> Ctrain = confusion_matrix(ytrain, yhat, 3)',\n",
       "  '',\n",
       "  '3×3 Array{Float64,2}:',\n",
       "  '40.0 0.0 0.0',\n",
       "  '',\n",
       "  '0.0 28.0 12.0',\n",
       "  '',\n",
       "  '0.0 6.0 34.0',\n",
       "  '',\n",
       "  'julia> error_train = error_rate(ytrain, yhat)',\n",
       "  '',\n",
       "  '0.15',\n",
       "  '',\n",
       "  \"julia> yhat = row_argmax(Xtest'*theta)\",\n",
       "  'julia> Ctest = confusion_matrix(ytest, yhat, 3)',\n",
       "  '',\n",
       "  '3×3 Array{Float64,2}:',\n",
       "  '10.0 0.0 0.0',\n",
       "  '',\n",
       "  '0.0 7.0 3.0',\n",
       "  '',\n",
       "  '0.0 2.0 8.0',\n",
       "  '',\n",
       "  'julia> error_test = error_rate(ytest, yhat)',\n",
       "  '',\n",
       "  '0.16666666666666666',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '120 14 Least squares classification',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Chapter 15',\n",
       "  '',\n",
       "  'Multi-objective least squares',\n",
       "  '',\n",
       "  '15.1 Multi-objective least squares',\n",
       "  '',\n",
       "  'Let’s write a functiont that solves the multi-objective least squares problem, with',\n",
       "  'given positive weights. The data are a list (or array) of coefficient matrices (of',\n",
       "  'possibly different heights) As, a matching list of (right-hand side) vectors bs, and',\n",
       "  'the weights, given as an array or list, lambdas.',\n",
       "  '',\n",
       "  'julia> function mols_solve(As,bs,lambdas)',\n",
       "  '',\n",
       "  'k = length(lambdas);',\n",
       "  '',\n",
       "  'Atil = vcat([sqrt(lambdas[i])*As[i] for i=1:k]...)',\n",
       "  '',\n",
       "  'btil = vcat([sqrt(lambdas[i])*bs[i] for i=1:k]...)',\n",
       "  '',\n",
       "  'return Atil \\\\ btil',\n",
       "  '',\n",
       "  'end',\n",
       "  '',\n",
       "  'Simple example. We use the function mols_solve to work out a bi-criterion',\n",
       "  'example similar to Figures 15.1, 15.2, and 15.3 in VMLS. We minimize the weighted',\n",
       "  'sum objective',\n",
       "  '',\n",
       "  'J1 + λJ2 = ‖A1x− b1‖2 + λ‖A2x− b2‖2',\n",
       "  '',\n",
       "  'for randomly chosen 10× 5 matrices A1, A2 and 10-vectors b1, b2. The expression',\n",
       "  'lambdas = 10 .^ linspace(-4,4,200) generates 200 values of λ ∈ [10−4, 104],',\n",
       "  'equally spaced on a logarithmic scale. The code creates the three plots in Fig-',\n",
       "  'ures 15.1, 15.2, and 15.3.',\n",
       "  '',\n",
       "  'julia> As = [randn(10,5), randn(10,5)];',\n",
       "  '',\n",
       "  'julia> bs = [randn(10), randn(10)];',\n",
       "  '',\n",
       "  'julia> N = 200;',\n",
       "  '',\n",
       "  'julia> lambdas = 10 .^ linspace(-4,4,200);',\n",
       "  '',\n",
       "  'julia> x = zeros(5,N); J1 = zeros(N); J2 = zeros(N);',\n",
       "  '',\n",
       "  'julia> for k = 1:N',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.15.1',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.15.2',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.15.3',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '122 15 Multi-objective least squares',\n",
       "  '',\n",
       "  '10-4 10-2 100 102 104 ',\n",
       "  '',\n",
       "  '-0.50',\n",
       "  '',\n",
       "  '-0.25',\n",
       "  '',\n",
       "  '0.00',\n",
       "  '',\n",
       "  '0.25',\n",
       "  '',\n",
       "  'lambda',\n",
       "  '',\n",
       "  'y1',\n",
       "  'y2',\n",
       "  'y3',\n",
       "  'y4',\n",
       "  'y5',\n",
       "  '',\n",
       "  'Figure 15.1 Weighted-sum least squares solution x̂(λ) as a function of λ for',\n",
       "  'a bi-criterion least squares problem with five variables.',\n",
       "  '',\n",
       "  'x[:,k] = mols_solve(As, bs, [1, lambdas[k]])',\n",
       "  '',\n",
       "  'J1[k] = norm(As[1]*x[:,k] - bs[1])^2',\n",
       "  '',\n",
       "  'J2[k] = norm(As[2]*x[:,k] - bs[2])^2',\n",
       "  '',\n",
       "  'end;',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> # plot solution versus lambda',\n",
       "  '',\n",
       "  'julia> plot(lambdas, x\\', xscale = :log10, xlabel = \"lambda\");',\n",
       "  'julia> plot!(xlims = (1e-4,1e4));',\n",
       "  '',\n",
       "  'julia> # plot two objectives versus lambda',\n",
       "  '',\n",
       "  'julia> plot(lambdas, J1, xscale = :log10, label = \"J_1\");',\n",
       "  '',\n",
       "  'julia> plot!(lambdas, J2, label = \"J_2\", xlabel = \"lambda\",',\n",
       "  '',\n",
       "  'xlims = (1e-4,1e4));',\n",
       "  '',\n",
       "  'julia> # plot trade-off curve',\n",
       "  '',\n",
       "  'julia> plot(J1, J2, xlabel=\"J1\", ylabel = \"J2\", legend=false);',\n",
       "  '',\n",
       "  'julia> # add (single-objective) end points to trade-off curve',\n",
       "  '',\n",
       "  'julia> x1 = As[1] \\\\ bs[1];',\n",
       "  '',\n",
       "  'julia> x2 = As[2] \\\\ bs[2];',\n",
       "  '',\n",
       "  'julia> J1 = [norm(As[1]*x1-bs[1])^2, norm(As[1]*x2-bs[1])^2];',\n",
       "  '',\n",
       "  'julia> J2 = [norm(As[2]*x1-bs[2])^2, norm(As[2]*x2-bs[2])^2];',\n",
       "  '',\n",
       "  'julia> scatter!(J1,J2);',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '15.1 Multi-objective least squares 123',\n",
       "  '',\n",
       "  '10-4 10-2 100 102 104 ',\n",
       "  '',\n",
       "  '5.0',\n",
       "  '',\n",
       "  '7.5',\n",
       "  '',\n",
       "  '10.0',\n",
       "  '',\n",
       "  '12.5',\n",
       "  '',\n",
       "  '15.0',\n",
       "  '',\n",
       "  '17.5',\n",
       "  '',\n",
       "  'lambda',\n",
       "  '',\n",
       "  'J_1',\n",
       "  'J_2',\n",
       "  '',\n",
       "  'Figure 15.2 Objective functions J1 = ‖A1x̂(λ)b1‖2 (blue line) and J2 =',\n",
       "  '‖A2x̂(λ)b2‖2 (red line) as functions of λ for the bi-criterion problem in fig-',\n",
       "  'ure 15.1.',\n",
       "  '',\n",
       "  '7.5 10.0 12.5 15.0 17.5',\n",
       "  '',\n",
       "  '5.0',\n",
       "  '',\n",
       "  '7.5',\n",
       "  '',\n",
       "  '10.0',\n",
       "  '',\n",
       "  '12.5',\n",
       "  '',\n",
       "  '15.0',\n",
       "  '',\n",
       "  '17.5',\n",
       "  '',\n",
       "  'J1',\n",
       "  '',\n",
       "  'J2',\n",
       "  '',\n",
       "  'Figure 15.3 Optimal trade-off curve for the bi-criterion least squares problem',\n",
       "  'of figures 15.1 and 15.2.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '124 15 Multi-objective least squares',\n",
       "  '',\n",
       "  '0 100 200 300',\n",
       "  '',\n",
       "  '10-2.4 ',\n",
       "  '',\n",
       "  '10-2.1 ',\n",
       "  '',\n",
       "  '10-1.8 ',\n",
       "  '',\n",
       "  '10-1.5 ',\n",
       "  '',\n",
       "  '10-1.2 ',\n",
       "  '',\n",
       "  'Figure 15.4 Hourly ozone level at Azusa, California, during the first 14 days',\n",
       "  'of July 2014 (California Environmental Protection Agency, Air Re- sources',\n",
       "  'Board, www.arb.ca.gov). Measurements start at 12AM on July 1st, and end',\n",
       "  'at 11PM on July 14. Note the large number of missing measure- ments. In',\n",
       "  'particular, all 4AM measurements are missing.',\n",
       "  '',\n",
       "  '15.2 Control',\n",
       "  '',\n",
       "  '15.3 Estimation and inversion',\n",
       "  '',\n",
       "  'Estimating a periodic time series. We consider the example of Figure 15.4 in',\n",
       "  'VMLS. We start by loading the data, as a vector with hourly ozone levels, for a',\n",
       "  'period of 14 days. Missing measurements have a value NaN (for Not a Number).',\n",
       "  'The plot command skips those values (Figure 15.4).',\n",
       "  '',\n",
       "  'julia> ozone = ozone_data(); # a vector of length 14*24 = 336',\n",
       "  '',\n",
       "  'julia> k = 14; N = k*24;',\n",
       "  '',\n",
       "  'julia> plot(1:N, ozone, yscale = :log10, marker = :circle,',\n",
       "  '',\n",
       "  'legend=false)',\n",
       "  '',\n",
       "  'Next we use the mols_solve function to make a periodic fit, for the values',\n",
       "  'λ = 1 and λ = 10. The Julia code isnan is used to find and discard the missing',\n",
       "  'measurements. The results are shown in Figures 15.5 and 15.6.',\n",
       "  '',\n",
       "  'julia> A = vcat( [eye(24) for i = 1:k]...)',\n",
       "  '',\n",
       "  'julia> # periodic difference matrix',\n",
       "  '',\n",
       "  'julia> D = -eye(24) + [zeros(23,1) eye(23); 1 zeros(1,23)];',\n",
       "  '',\n",
       "  'julia> ind = [k for k in 1:length(ozone) if !isnan(ozone[k])];',\n",
       "  '',\n",
       "  'julia> As = [A[ind,:], D]',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.15.4',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '15.4 Regularized data fitting 125',\n",
       "  '',\n",
       "  '0 100 200 300',\n",
       "  '',\n",
       "  '10-2.4 ',\n",
       "  '',\n",
       "  '10-2.1 ',\n",
       "  '',\n",
       "  '10-1.8 ',\n",
       "  '',\n",
       "  '10-1.5 ',\n",
       "  '',\n",
       "  '10-1.2 ',\n",
       "  '',\n",
       "  'Figure 15.5 Smooth periodic least squares fit to logarithmically transformed',\n",
       "  'measurements, using λ = 1.',\n",
       "  '',\n",
       "  'julia> bs = [log.(ozone[ind]), zeros(24)]',\n",
       "  '',\n",
       "  'julia> # solution for lambda = 1',\n",
       "  '',\n",
       "  'julia> x = mols_solve( As, bs, [1, 1])',\n",
       "  '',\n",
       "  'julia> scatter(1:N, ozone, yscale = :log10, legend=false)',\n",
       "  '',\n",
       "  'julia> plot!(1:N, vcat([exp.(x) for i = 1:k]...))',\n",
       "  '',\n",
       "  'julia> # solution for lambda = 100',\n",
       "  '',\n",
       "  'julia> x = mols_solve( As, bs, [1, 100])',\n",
       "  '',\n",
       "  'julia> scatter(1:N, ozone, yscale = :log10, legend=false)',\n",
       "  '',\n",
       "  'julia> plot!(1:N, vcat([exp.(x) for i = 1:k]...))',\n",
       "  '',\n",
       "  '15.4 Regularized data fitting',\n",
       "  '',\n",
       "  'Example. Next we consider the small regularized data fitting example of page 329',\n",
       "  'of VMLS. We fit a model',\n",
       "  '',\n",
       "  'f̂(x) =',\n",
       "  '',\n",
       "  '5∑',\n",
       "  'k=1',\n",
       "  '',\n",
       "  'θkfk(x)',\n",
       "  '',\n",
       "  'with basis functions f1(x) = 1 and fk+1(x) = sin(ωkx + φk) for k = 1, . . . , 4 to',\n",
       "  'N = 20 data points. We use the values of ωk, φk given in the text. We fit the',\n",
       "  'model by solving a sequence of regularized least squares problems with objective',\n",
       "  '',\n",
       "  'N∑',\n",
       "  'i=1',\n",
       "  '',\n",
       "  '(y(i) −',\n",
       "  '5∑',\n",
       "  'k=1',\n",
       "  '',\n",
       "  'θkfk(x',\n",
       "  '(i)))2 + λ',\n",
       "  '',\n",
       "  '5∑',\n",
       "  'k=2',\n",
       "  '',\n",
       "  'θ2k.',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.367',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '126 15 Multi-objective least squares',\n",
       "  '',\n",
       "  '0 100 200 300',\n",
       "  '',\n",
       "  '10-2.4 ',\n",
       "  '',\n",
       "  '10-2.1 ',\n",
       "  '',\n",
       "  '10-1.8 ',\n",
       "  '',\n",
       "  '10-1.5 ',\n",
       "  '',\n",
       "  '10-1.2 ',\n",
       "  '',\n",
       "  'Figure 15.6 Smooth periodic least squares fit to logarithmically transformed',\n",
       "  'measurements, using λ = 100.',\n",
       "  '',\n",
       "  'The two plots are shown in Figures 15.7 and 15.8.',\n",
       "  '',\n",
       "  'julia> # Import data as vectors xtrain, ytrain, xtest, ytest',\n",
       "  '',\n",
       "  'julia> D = regularized_fit_data();',\n",
       "  '',\n",
       "  'julia> xtrain = D[\"xtrain\"]; ytrain = D[\"ytrain\"];',\n",
       "  '',\n",
       "  'julia> xtest = D[\"xtest\"]; ytest = D[\"ytest\"];',\n",
       "  '',\n",
       "  'julia> N = length(ytrain);',\n",
       "  '',\n",
       "  'julia> Ntest = length(ytest);',\n",
       "  '',\n",
       "  'julia> p = 5;',\n",
       "  '',\n",
       "  'julia> omega = [ 13.69; 3.55; 23.25; 6.03 ];',\n",
       "  '',\n",
       "  'julia> phi = [ 0.21; 0.02; -1.87; 1.72 ];',\n",
       "  '',\n",
       "  \"julia> A = hcat(ones(N), sin.(xtrain*omega' + ones(N)*phi'));\",\n",
       "  'julia> Atest = hcat(ones(Ntest),',\n",
       "  '',\n",
       "  \"sin.(xtest*omega' + ones(Ntest)*phi'));\",\n",
       "  'julia> npts = 100;',\n",
       "  '',\n",
       "  'julia> lambdas = 10 .^ linspace(-6,6,npts);',\n",
       "  '',\n",
       "  'julia> err_train = zeros(npts);',\n",
       "  '',\n",
       "  'julia> err_test = zeros(npts);',\n",
       "  '',\n",
       "  'julia> thetas = zeros(p,npts);',\n",
       "  '',\n",
       "  'julia> for k = 1:npts',\n",
       "  '',\n",
       "  'theta = mols_solve([ A, [zeros(p-1) eye(p-1)]],',\n",
       "  '',\n",
       "  '[ ytrain, zeros(p-1) ], [1, lambdas[k]])',\n",
       "  '',\n",
       "  'err_train[k] = rms(ytrain - A*theta);',\n",
       "  '',\n",
       "  'err_test[k] = rms(ytest - Atest*theta);',\n",
       "  '',\n",
       "  'thetas[:,k] = theta;',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '15.5 Complexity 127',\n",
       "  '',\n",
       "  '10-6 10-4 10-2 100 102 104 106 ',\n",
       "  '',\n",
       "  '0.2',\n",
       "  '',\n",
       "  '0.4',\n",
       "  '',\n",
       "  '0.6',\n",
       "  '',\n",
       "  '0.8',\n",
       "  '',\n",
       "  '1.0',\n",
       "  '',\n",
       "  'lambda',\n",
       "  '',\n",
       "  'R',\n",
       "  'M',\n",
       "  '',\n",
       "  'S',\n",
       "  ' e',\n",
       "  '',\n",
       "  'rr',\n",
       "  'or',\n",
       "  '',\n",
       "  'Train',\n",
       "  'Test',\n",
       "  '',\n",
       "  'Figure 15.7 RMS training and test errors as a function of the regularization',\n",
       "  'parameter λ.',\n",
       "  '',\n",
       "  'end;',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> # Plot RMS errors',\n",
       "  '',\n",
       "  'julia> plot(lambdas, err_train, xscale = :log10, label = \"Train\")',\n",
       "  '',\n",
       "  'julia> plot!(lambdas, err_test, xscale = :log10, label = \"Test\")',\n",
       "  '',\n",
       "  'julia> plot!(xlabel = \"lambda\", ylabel = \"RMS error\",',\n",
       "  '',\n",
       "  'xlim = (1e-6, 1e6));',\n",
       "  '',\n",
       "  'julia> # Plot coefficients',\n",
       "  '',\n",
       "  \"julia> plot(lambdas, thetas', xscale = :log10)\",\n",
       "  'julia> plot!(xlabel = \"lambda\", xlim = (1e-6, 1e6));',\n",
       "  '',\n",
       "  '15.5 Complexity',\n",
       "  '',\n",
       "  'The kernel trick. Let’s check the kernel trick, described in §15.5.2, to find x̂, the',\n",
       "  'minimizer of',\n",
       "  '',\n",
       "  '‖Ax− b‖2 + λ‖x− xdes‖2,',\n",
       "  'where A is an m×n matrix and λ > 0. We’ll compute x̂ two ways. First, the näıve',\n",
       "  'way, and then, using the kernel trick. We use the fact that if[',\n",
       "  '',\n",
       "  'AT√',\n",
       "  'λI',\n",
       "  '',\n",
       "  ']',\n",
       "  '= QR,',\n",
       "  '',\n",
       "  'then',\n",
       "  '(AAT + λI)−1 = (RTQTQR)−1 = R−1R−T .',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#subsection.15.5.2',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '128 15 Multi-objective least squares',\n",
       "  '',\n",
       "  '10-6 10-4 10-2 100 102 104 106 ',\n",
       "  '-1.5',\n",
       "  '',\n",
       "  '-1.0',\n",
       "  '',\n",
       "  '-0.5',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '0.5',\n",
       "  '',\n",
       "  '1.0',\n",
       "  '',\n",
       "  '1.5',\n",
       "  '',\n",
       "  'lambda',\n",
       "  '',\n",
       "  'y1',\n",
       "  'y2',\n",
       "  'y3',\n",
       "  'y4',\n",
       "  'y5',\n",
       "  '',\n",
       "  'Figure 15.8 The regularization path.',\n",
       "  '',\n",
       "  'julia> m = 100; n = 5000;',\n",
       "  '',\n",
       "  'julia> A = randn(m,n); b = randn(m); xdes = randn(n);',\n",
       "  '',\n",
       "  'julia> lam = 2.0;',\n",
       "  '',\n",
       "  'julia> # Find x that minimizes ||Ax-b||^2 + lambda ||x||^2',\n",
       "  '',\n",
       "  'julia> @time xhat1 = [A; sqrt(lam)*eye(n)] \\\\ [b; sqrt(lam)*xdes];',\n",
       "  '',\n",
       "  '23.447045 seconds (40.08 k allocations: 1.130 GiB, 0.83% gc time)',\n",
       "  '',\n",
       "  'julia> # Now use kernel trick',\n",
       "  '',\n",
       "  'julia> @time begin',\n",
       "  '',\n",
       "  \"Q, R = qr([A' ; sqrt(lam)*eye(m)]);\",\n",
       "  'Q = Matrix(Q);',\n",
       "  '',\n",
       "  \"xhat2 = A' * (R \\\\ (R' \\\\ (b-A*xdes))) + xdes;\",\n",
       "  'end;',\n",
       "  '',\n",
       "  '0.025105 seconds (42 allocations: 12.114 MiB)',\n",
       "  '',\n",
       "  'julia> norm(xhat1-xhat2)',\n",
       "  '',\n",
       "  '1.2742623007481903e-13',\n",
       "  '',\n",
       "  'The näıve method requires the factorization of a 5100×5100 matrix. In the second',\n",
       "  'method we factor a matrix of size 5100× 100.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Chapter 16',\n",
       "  '',\n",
       "  'Constrained least squares',\n",
       "  '',\n",
       "  '16.1 Constrained least squares problem',\n",
       "  '',\n",
       "  'In the examples in this section, we use the cls_solve function, given later, to find',\n",
       "  'the constrained least squares solution.',\n",
       "  '',\n",
       "  'Piecewise polynomial. We fit a function f̂ : R → R to some given data, where',\n",
       "  'f̂(x) = p(x) for x ≤ a and f̂(x) = q(x) for x > a, subject to p(a) = q(a) and',\n",
       "  'p′(a) = q′(a), i.e., the two polynomials have matching value and slope at the',\n",
       "  'knot point a. We have data points x1, . . . , xM ≤ a and xM+1, . . . , xN > a and',\n",
       "  'corresponding values y1, . . . , yN . In the example we take a = 0, polynomials p and',\n",
       "  'q of degree 3, and N = 2M = 140. The code creates a figure similar to Figure 16.1',\n",
       "  'of VMLS (Figure 16.1). We use the vandermonde function from page 58.',\n",
       "  '',\n",
       "  'julia> M = 70; N = 2*M;',\n",
       "  '',\n",
       "  'julia> xleft = rand(M) .- 1; xright = rand(M);',\n",
       "  '',\n",
       "  'julia> x = [xleft; xright]',\n",
       "  '',\n",
       "  'julia> y = x.^3 - x + 0.4 ./ (1 .+ 25*x.^2) + 0.05*randn(N);',\n",
       "  '',\n",
       "  'julia> n = 4;',\n",
       "  '',\n",
       "  'julia> A = [ vandermonde(xleft,n) zeros(M,n);',\n",
       "  '',\n",
       "  'zeros(M,n) vandermonde(xright,n)]',\n",
       "  '',\n",
       "  'julia> b = y;',\n",
       "  '',\n",
       "  'julia> C = [1 zeros(1,n-1) -1 zeros(1,n-1);',\n",
       "  '',\n",
       "  '0 1 zeros(1,n-2) 0 -1 zeros(1,n-2)];',\n",
       "  '',\n",
       "  'julia> d = zeros(2);',\n",
       "  '',\n",
       "  'julia> theta = cls_solve(A, b, C, d);',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> # Evaluate and plot for 200 equidistant points on each side.',\n",
       "  '',\n",
       "  'julia> Npl = 200;',\n",
       "  '',\n",
       "  'julia> xpl_left = linspace(-1, 0, Npl);',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.16.1',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '130 16 Constrained least squares',\n",
       "  '',\n",
       "  '-1.0 -0.5 0.0 0.5 1.0',\n",
       "  '',\n",
       "  '-0.4',\n",
       "  '',\n",
       "  '-0.2',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '0.2',\n",
       "  '',\n",
       "  '0.4',\n",
       "  '',\n",
       "  'Figure 16.1 Least squares fit of two cubic polynomials to 140 points, with',\n",
       "  'continuity constraints p(0) = q(0) and p′(0) = q′(0).',\n",
       "  '',\n",
       "  'julia> ypl_left = vandermonde(xpl_left, 4)*theta[1:n];',\n",
       "  '',\n",
       "  'julia> xpl_right = linspace(0, 1, Npl);',\n",
       "  '',\n",
       "  'julia> ypl_right = vandermonde(xpl_right, 4)*theta[n+1:end];',\n",
       "  '',\n",
       "  'julia> scatter(x,y, legend=false)',\n",
       "  '',\n",
       "  'julia> plot!(xpl_left, ypl_left)',\n",
       "  '',\n",
       "  'julia> plot!(xpl_right, ypl_right)',\n",
       "  '',\n",
       "  'Advertising budget. We continue the advertising example of page 96 and add a',\n",
       "  'total budget constraint 1T s = 1284.',\n",
       "  '',\n",
       "  'julia> cls_solve(R, 1e3*ones(m), ones(1,n), 1284)',\n",
       "  '',\n",
       "  '3×1 Array{Float64,2}:',\n",
       "  '315.16818459234986',\n",
       "  '',\n",
       "  '109.86643348012254',\n",
       "  '',\n",
       "  '858.9653819275276',\n",
       "  '',\n",
       "  'Minimum norm force sequence. We compute the smallest sequence of ten forces,',\n",
       "  'each applied for one second to a unit frictionless mass originally at rest, that moves',\n",
       "  'the mass position one with zero velocity (VMLS page 343).',\n",
       "  '',\n",
       "  'julia> A = eye(10); b = zeros(10);',\n",
       "  '',\n",
       "  \"julia> C = [ones(1,10); (9.5:-1:0.5)']\",\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.373',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '16.2 Solution 131',\n",
       "  '',\n",
       "  '2×10 Array{Float64,2}:',\n",
       "  '1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0 1.0',\n",
       "  '',\n",
       "  '9.5 8.5 7.5 6.5 5.5 4.5 3.5 2.5 1.5 0.5',\n",
       "  '',\n",
       "  'julia> d = [0,1];',\n",
       "  '',\n",
       "  'julia> fln = cls_solve(A,b,C,d)',\n",
       "  '',\n",
       "  '10×1 Array{Float64,2}:',\n",
       "  '0.0545455',\n",
       "  '',\n",
       "  '0.0424242',\n",
       "  '',\n",
       "  '0.030303',\n",
       "  '',\n",
       "  '0.0181818',\n",
       "  '',\n",
       "  '0.00606061',\n",
       "  '',\n",
       "  '-0.00606061',\n",
       "  '',\n",
       "  '-0.0181818',\n",
       "  '',\n",
       "  '-0.030303',\n",
       "  '',\n",
       "  '-0.0424242',\n",
       "  '',\n",
       "  '-0.0545455',\n",
       "  '',\n",
       "  '16.2 Solution',\n",
       "  '',\n",
       "  'Let’s implement the function cls_solve_kkt, which finds the constrained least',\n",
       "  'squares solution by forming the KKT system and solving it. We allow the b and',\n",
       "  'd to be matrices, so one function call can solve mutiple problems with the same A',\n",
       "  'and C.',\n",
       "  '',\n",
       "  'julia> function cls_solve_kkt(A,b,C,d)',\n",
       "  '',\n",
       "  'm, n = size(A)',\n",
       "  '',\n",
       "  'p, n = size(C)',\n",
       "  '',\n",
       "  \"G = A'*A # Gram matrix\",\n",
       "  \"KKT = [2*G C'; C zeros(p,p)] # KKT matrix\",\n",
       "  \"xzhat = KKT \\\\ [2*A'*b; d]\",\n",
       "  'return xzhat[1:n,:]',\n",
       "  '',\n",
       "  'end;',\n",
       "  '',\n",
       "  'julia> A = randn(10,5); b = randn(10);',\n",
       "  '',\n",
       "  'julia> C = randn(2,5); d = randn(2);',\n",
       "  '',\n",
       "  'julia> x = cls_solve_kkt(A,b,C,d);',\n",
       "  '',\n",
       "  'julia> C*x - d # Check residual small',\n",
       "  '',\n",
       "  '2×1 Array{Float64,2}:',\n",
       "  '-5.551115123125783e-17',\n",
       "  '',\n",
       "  '-1.6653345369377348e-16',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '132 16 Constrained least squares',\n",
       "  '',\n",
       "  '16.3 Solving contrained least squares problems',\n",
       "  '',\n",
       "  'Solving constrained least squares via QR. Let’s implement VMLS algorithm 16.1',\n",
       "  'and then check it against our method above, which forms and solves the KKT',\n",
       "  'system.',\n",
       "  '',\n",
       "  'julia> function cls_solve(A,b,C,d)',\n",
       "  '',\n",
       "  'm, n = size(A)',\n",
       "  '',\n",
       "  'p, n = size(C)',\n",
       "  '',\n",
       "  'Q, R = qr([A; C])',\n",
       "  '',\n",
       "  'Q = Matrix(Q)',\n",
       "  '',\n",
       "  'Q1 = Q[1:m,:]',\n",
       "  '',\n",
       "  'Q2 = Q[m+1:m+p,:]',\n",
       "  '',\n",
       "  \"Qtil, Rtil = qr(Q2')\",\n",
       "  'Qtil = Matrix(Qtil)',\n",
       "  '',\n",
       "  \"w = Rtil \\\\ (2*Qtil'*Q1'*b - 2*(Rtil'\\\\d))\",\n",
       "  \"return xhat = R \\\\ (Q1'*b - Q2'*w/2)\",\n",
       "  'end',\n",
       "  '',\n",
       "  'julia> # check with KKT method',\n",
       "  '',\n",
       "  'julia> m = 10; n = 5; p = 2;',\n",
       "  '',\n",
       "  'julia> A = randn(m,n); b = randn(m); C = randn(p,n); d = randn(p);',\n",
       "  '',\n",
       "  'julia> xKKT = cls_solve_kkt(A,b,C,d);',\n",
       "  '',\n",
       "  'julia> xQR = cls_solve(A,b,C,d);',\n",
       "  '',\n",
       "  'julia> norm(xKKT-xQR)',\n",
       "  '',\n",
       "  '1.4931525882746458e-15',\n",
       "  '',\n",
       "  'The function cls_solve is included in the VMLS package.',\n",
       "  '',\n",
       "  'Sparse constrained least squares. Let’s form and solve the system of linear equa-',\n",
       "  'tions in VMSL (16.11), and compare it to our basic method for constrained least',\n",
       "  'squares. This formulation will result in a sparse set of equations to solve if A and',\n",
       "  'C are sparse. (The code below just checks that the two methods agree; it does not',\n",
       "  'use sparsity. Unlike the earlier cls_solve, it assumes b and d are vectors.)',\n",
       "  '',\n",
       "  'julia> function cls_solve_sparse(A,b,C,d)',\n",
       "  '',\n",
       "  'm, n = size(A)',\n",
       "  '',\n",
       "  'p, n = size(C)',\n",
       "  '',\n",
       "  \"bigA = [ zeros(n,n) A' C';\",\n",
       "  'A -I/2 zeros(m,p) ;',\n",
       "  '',\n",
       "  'C zeros(p,m) zeros(p,p) ]',\n",
       "  '',\n",
       "  'xyzhat = bigA \\\\ [zeros(n) ; b ; d]',\n",
       "  '',\n",
       "  'return xhat = xyzhat[1:n]',\n",
       "  '',\n",
       "  'end',\n",
       "  '',\n",
       "  'julia> m = 100; n = 50; p = 10;',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#algorithmctr.16.1',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#equation.16.3.11',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '16.3 Solving contrained least squares problems 133',\n",
       "  '',\n",
       "  'julia> A = randn(m,n); b = randn(m); C = randn(p,n); d = randn(p);',\n",
       "  '',\n",
       "  'julia> x1 = cls_solve(A,b,C,d);',\n",
       "  '',\n",
       "  'julia> x2 = cls_solve_sparse(A,b,C,d);',\n",
       "  '',\n",
       "  'julia> norm(x1-x2)',\n",
       "  '',\n",
       "  '1.3344943251376455e-14',\n",
       "  '',\n",
       "  'Solving least norm problem. In Julia, the backslash operator is used to find the',\n",
       "  'least norm solution x̂ of an under-determined set of equations Cx = d. Thus the',\n",
       "  'backslash operator is overloaded to solve linear equations with a square coefficient',\n",
       "  'matrix, find a least squares approximate solution when the coefficient matrix is',\n",
       "  'tall, and find the least norm solution when the coefficient matrix is wide.',\n",
       "  '',\n",
       "  'Let’s solve a least norm problem using several methods, to check that they',\n",
       "  'agree.',\n",
       "  '',\n",
       "  'julia> p = 50; n = 500;',\n",
       "  '',\n",
       "  'julia> C = randn(p,n); d = randn(p);',\n",
       "  '',\n",
       "  'julia> x1 = C\\\\d; # Solve using backslash',\n",
       "  '',\n",
       "  'julia> # Solve using cls_solve, which uses KKT system',\n",
       "  '',\n",
       "  'julia> x2 = cls_solve(eye(n), zeros(n), C, d);',\n",
       "  '',\n",
       "  'julia> x3 = pinv(C)*d; # Using pseudo-inverse',\n",
       "  '',\n",
       "  'julia> norm(x1-x2)',\n",
       "  '',\n",
       "  '5.584943800596077e-15',\n",
       "  '',\n",
       "  'julia> norm(x2-x3)',\n",
       "  '',\n",
       "  '5.719694159427276e-15',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '134 16 Constrained least squares',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Chapter 17',\n",
       "  '',\n",
       "  'Constrained least squares',\n",
       "  'applications',\n",
       "  '',\n",
       "  '17.1 Portfolio optimization',\n",
       "  '',\n",
       "  'Compounded portfolio value. The cumulative value of a portfolio from a return',\n",
       "  'time series vector r, starting from the traditional value of $10000, is given by the',\n",
       "  'value times series vector v, where',\n",
       "  '',\n",
       "  'vt = 10000(1 + r1) · · · (1 + rt−1), t = 1, . . . , T.',\n",
       "  '',\n",
       "  'In other words, we form the cumulative product of the vector with entries 1 + rt.',\n",
       "  'Julia has a built-in function that does this, cumprod.',\n",
       "  '',\n",
       "  'julia> # Portfolio value with re-investment, return time series r',\n",
       "  '',\n",
       "  'julia> cum_value(r) = 10000 * cumprod(1 .+ r)',\n",
       "  '',\n",
       "  'julia> T = 250; # One year’s worth of trading days',\n",
       "  '',\n",
       "  'julia> # Generate random returns sequence with',\n",
       "  '',\n",
       "  'julia> # 10% annualized return, 5% annualized risk',\n",
       "  '',\n",
       "  'julia> mu = 0.10/250; sigma = 0.05/sqrt(250);',\n",
       "  '',\n",
       "  'julia> r = mu .+ sigma*randn(T);',\n",
       "  '',\n",
       "  'julia> v = cum_value(r);',\n",
       "  '',\n",
       "  'julia> # compare final value (compounded) and average return',\n",
       "  '',\n",
       "  'julia> v[T] , v[1]*(1+sum(r))',\n",
       "  '',\n",
       "  '(10313.854295827463, 10348.11585318395)',\n",
       "  '',\n",
       "  'julia> # plot cumulative value over the year',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> plot(1:T, v, legend=false)',\n",
       "  '',\n",
       "  'julia> plot!( xlabel = \"t\", ylabel = \"v_t\")',\n",
       "  '',\n",
       "  'The resulting figure for a particular choice of r is shown in Figure 17.1.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '136 17 Constrained least squares applications',\n",
       "  '',\n",
       "  '0 50 100 150 200 250',\n",
       "  '',\n",
       "  '10000',\n",
       "  '',\n",
       "  '10100',\n",
       "  '',\n",
       "  '10200',\n",
       "  '',\n",
       "  '10300',\n",
       "  '',\n",
       "  '10400',\n",
       "  '',\n",
       "  '10500',\n",
       "  '',\n",
       "  't',\n",
       "  '',\n",
       "  'v_',\n",
       "  't',\n",
       "  '',\n",
       "  'Figure 17.1 Total portfolio value over time.',\n",
       "  '',\n",
       "  'Portfolio optimization. We define a function port_opt that evaluates the solu-',\n",
       "  'tion (17.3) of the constrained least squares problem (17.2) in VMLS, and apply to',\n",
       "  'the return data in VMLS Section 17.1.3.',\n",
       "  '',\n",
       "  'julia> function port_opt(R,rho)',\n",
       "  '',\n",
       "  'T, n = size(R)',\n",
       "  '',\n",
       "  \"mu = sum(R, dims=1)'/T\",\n",
       "  \"KKT = [ 2*R'*R ones(n) mu; ones(n)' 0 0; mu' 0 0]\",\n",
       "  'wz1z2 = KKT \\\\ [2*rho*T*mu; 1; rho]',\n",
       "  '',\n",
       "  'w = wz1z2[1:n]',\n",
       "  '',\n",
       "  'return w',\n",
       "  '',\n",
       "  'end;',\n",
       "  '',\n",
       "  'julia> R, Rtest = portfolio_data();',\n",
       "  '',\n",
       "  'julia> T, n = size(R)',\n",
       "  '',\n",
       "  '(2000, 20)',\n",
       "  '',\n",
       "  'julia> rho = 0.10/250; # Ask for 10% annual return',\n",
       "  '',\n",
       "  'julia> w = port_opt(R,rho);',\n",
       "  '',\n",
       "  'julia> r = R*w; # Portfolio return time series',\n",
       "  '',\n",
       "  'julia> pf_return = 250*avg(r)',\n",
       "  '',\n",
       "  '0.10000000000000003',\n",
       "  '',\n",
       "  'julia> pf_risk = sqrt(250)*stdev(r)',\n",
       "  '',\n",
       "  '0.0865018308685463',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> plot(1:T, cum_value(r), label= \"10%\")',\n",
       "  '',\n",
       "  'This produces the curve labeled “10%” in Figure 17.2. We also included the plots',\n",
       "  'for 20% and 40% annual return, and for the 1/n portfolio w = (1/n)1.',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#equation.17.1.3',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#equation.17.1.2',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#subsection.17.1.3',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '17.2 Linear quadratic control 137',\n",
       "  '',\n",
       "  '0 500 1000 1500 2000',\n",
       "  '',\n",
       "  '2.50×104',\n",
       "  '',\n",
       "  '5.00×104',\n",
       "  '',\n",
       "  '7.50×104',\n",
       "  '',\n",
       "  '1.00×105',\n",
       "  '',\n",
       "  '1.25×105',\n",
       "  '10%',\n",
       "  '20%',\n",
       "  '40%',\n",
       "  '1/n',\n",
       "  '',\n",
       "  'Figure 17.2 Total value over time for four portfolios: the Pareto optimal',\n",
       "  'portfolios with 10%, 20%, and 40% return, and the uniform portfolio. The',\n",
       "  'total value is computed using the 2000× 20 daily return matrix R.',\n",
       "  '',\n",
       "  '17.2 Linear quadratic control',\n",
       "  '',\n",
       "  'We implement linear quadratic control, as described in VMLS §17.2, for a time-',\n",
       "  'invariant system with matrices A, B, and C.',\n",
       "  '',\n",
       "  'Kronecker product. To create the big matrices Ã and C̃, we need to define block',\n",
       "  'diagonal matrices with the same matrix repeated a number of times along the',\n",
       "  'diagonal. There are many ways to do this in Julia. One of the simplest ways',\n",
       "  'uses the kron function, for the Kronecker product of two matrices. The Kronecker',\n",
       "  'product of an m × n matrix G and a p × q matrix H is defined as the mp × nq',\n",
       "  'block matrix \\uf8ee\\uf8ef\\uf8ef\\uf8ef\\uf8f0',\n",
       "  '',\n",
       "  'G11H G12H · · · G1nH',\n",
       "  'G21H G22H · · · G2nH',\n",
       "  '',\n",
       "  '...',\n",
       "  '...',\n",
       "  '',\n",
       "  '...',\n",
       "  'Gm1H Gm2H · · · GmnH',\n",
       "  '',\n",
       "  '\\uf8f9\\uf8fa\\uf8fa\\uf8fa\\uf8fb .',\n",
       "  'It is computed in Julia as kron(G,H). If G is an n× n identity matrix, we obtain',\n",
       "  'the block diagonal matrix with H repeated n times on the diagonal.',\n",
       "  '',\n",
       "  'julia> H = randn(2,2)',\n",
       "  '',\n",
       "  '2x2 Array{Float64,2}:',\n",
       "  '',\n",
       "  '1.73065 -1.33313',\n",
       "  '',\n",
       "  '-1.52245 0.0200201',\n",
       "  '',\n",
       "  'julia> kron(eye(3),H)',\n",
       "  '',\n",
       "  '6×6 Array{Float64,2}:',\n",
       "  '1.73065 -1.33313 0.0 -0.0 0.0 -0.0',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section.17.2',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '138 17 Constrained least squares applications',\n",
       "  '',\n",
       "  '-1.52245 0.0200201 -0.0 0.0 -0.0 0.0',\n",
       "  '',\n",
       "  '0.0 -0.0 1.73065 -1.33313 0.0 -0.0',\n",
       "  '',\n",
       "  '-0.0 0.0 -1.52245 0.0200201 -0.0 0.0',\n",
       "  '',\n",
       "  '0.0 -0.0 0.0 -0.0 1.73065 -1.33313',\n",
       "  '',\n",
       "  '-0.0 0.0 -0.0 0.0 -1.52245 0.0200201',\n",
       "  '',\n",
       "  'An alternative method uses the Julia cat function for constructing block matrices:',\n",
       "  '',\n",
       "  'julia> cat([H for k=1:3]..., dims=(1,2))',\n",
       "  '',\n",
       "  '6×6 Array{Float64,2}:',\n",
       "  '1.73065 -1.33313 0.0 -0.0 0.0 -0.0',\n",
       "  '',\n",
       "  '-1.52245 0.0200201 -0.0 0.0 -0.0 0.0',\n",
       "  '',\n",
       "  '0.0 -0.0 1.73065 -1.33313 0.0 -0.0',\n",
       "  '',\n",
       "  '-0.0 0.0 -1.52245 0.0200201 -0.0 0.0',\n",
       "  '',\n",
       "  '0.0 -0.0 0.0 -0.0 1.73065 -1.33313',\n",
       "  '',\n",
       "  '-0.0 0.0 -0.0 0.0 -1.52245 0.0200201',\n",
       "  '',\n",
       "  'Linear quadratic control example. We start by writing a function lqr that con-',\n",
       "  'structs and solves the constrained least squares problem for linear quadratic control.',\n",
       "  'The function returns three arrays',\n",
       "  '',\n",
       "  'x = [ x[1], x[2], ..., x[T] ],',\n",
       "  'u = [ u[1], u[2], ..., u[T-1] ],',\n",
       "  'y = [ y[1], y[2], ..., y[T] ].',\n",
       "  '',\n",
       "  'The first two contain the optimal solution of the problem. The third array contains',\n",
       "  'yt = Cxt.',\n",
       "  '',\n",
       "  'We allow the input arguments x_init and x_des to be matrices, so we can solve',\n",
       "  'the same problem for different pairs of initial and end states, with one function call.',\n",
       "  'If the number of columns in x_init and x_des is q, then the entries of the three',\n",
       "  'output sequences x, u, y are matrices with q columns. The ith columns are the',\n",
       "  'solution for the initial and end states specified in the ith columns of x_init and',\n",
       "  'x_des.',\n",
       "  '',\n",
       "  'function lqr(A,B,C,x_init,x_des,T,rho)',\n",
       "  '',\n",
       "  'n = size(A,1)',\n",
       "  '',\n",
       "  'm = size(B,2)',\n",
       "  '',\n",
       "  'p = size(C,1)',\n",
       "  '',\n",
       "  'q = size(x_init,2)',\n",
       "  '',\n",
       "  'Atil = [ kron(eye(T), C) zeros(p*T,m*(T-1)) ;',\n",
       "  '',\n",
       "  'zeros(m*(T-1), n*T) sqrt(rho)*eye(m*(T-1)) ]',\n",
       "  '',\n",
       "  'btil = zeros(p*T + m*(T-1), q)',\n",
       "  '',\n",
       "  '# We’ll construct Ctilde bit by bit',\n",
       "  '',\n",
       "  'Ctil11 = [ kron(eye(T-1), A) zeros(n*(T-1),n) ] -',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '17.2 Linear quadratic control 139',\n",
       "  '',\n",
       "  '[ zeros(n*(T-1), n) eye(n*(T-1)) ]',\n",
       "  '',\n",
       "  'Ctil12 = kron(eye(T-1), B)',\n",
       "  '',\n",
       "  'Ctil21 = [eye(n) zeros(n,n*(T-1)); zeros(n,n*(T-1)) eye(n)]',\n",
       "  '',\n",
       "  'Ctil22 = zeros(2*n,m*(T-1))',\n",
       "  '',\n",
       "  'Ctil = [Ctil11 Ctil12; Ctil21 Ctil22]',\n",
       "  '',\n",
       "  'dtil = [zeros(n*(T-1), q); x_init; x_des]',\n",
       "  '',\n",
       "  'z = cls_solve(Atil,btil,Ctil,dtil)',\n",
       "  '',\n",
       "  'x = [z[(i-1)*n+1:i*n,:] for i=1:T]',\n",
       "  '',\n",
       "  'u = [z[n*T+(i-1)*m+1 : n*T+i*m, :] for i=1:T-1]',\n",
       "  '',\n",
       "  'y = [C*xt for xt in x]',\n",
       "  '',\n",
       "  'return x, u, y',\n",
       "  '',\n",
       "  'end;',\n",
       "  '',\n",
       "  'We apply the function to the example in §17.2.1.',\n",
       "  '',\n",
       "  'julia> A = [ 0.855 1.161 0.667;',\n",
       "  '',\n",
       "  '0.015 1.073 0.053;',\n",
       "  '',\n",
       "  '-0.084 0.059 1.022 ];',\n",
       "  '',\n",
       "  'julia> B = [-0.076; -0.139; 0.342 ];',\n",
       "  '',\n",
       "  'julia> C = [ 0.218 -3.597 -1.683 ];',\n",
       "  '',\n",
       "  'julia> n = 3; p = 1; m = 1;',\n",
       "  '',\n",
       "  'julia> x_init = [0.496; -0.745; 1.394];',\n",
       "  '',\n",
       "  'julia> x_des = zeros(n,1);',\n",
       "  '',\n",
       "  'We first plot the open-loop response of VMLS figure 17.4 in figure 17.3.',\n",
       "  '',\n",
       "  'julia> T = 100;',\n",
       "  '',\n",
       "  'julia> yol = zeros(T,1);',\n",
       "  '',\n",
       "  'julia> Xol = [ x_init zeros(n, T-1) ];',\n",
       "  '',\n",
       "  'julia> for k=1:T-1',\n",
       "  '',\n",
       "  'Xol[:,k+1] = A*Xol[:,k];',\n",
       "  '',\n",
       "  'end;',\n",
       "  '',\n",
       "  'julia> yol = C*Xol;',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  \"julia> plot(1:T, yol', legend = false)\",\n",
       "  '',\n",
       "  'We then solve the linear quadratic control problem with T = 100 and ρ = 0.2. The',\n",
       "  'result is shown in the second row of VMLS figure 17.6 and in figure 17.4.',\n",
       "  '',\n",
       "  'julia> rho = 0.2;',\n",
       "  '',\n",
       "  'julia> T = 100;',\n",
       "  '',\n",
       "  'julia> x, u, y = lqr(A,B,C,x_init,x_des,T,rho)',\n",
       "  '',\n",
       "  'julia> J_input = norm(u)^2',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#subsection.17.2.1',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.17.4',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.17.6',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '140 17 Constrained least squares applications',\n",
       "  '',\n",
       "  '0 25 50 75 100',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '0.1',\n",
       "  '',\n",
       "  '0.2',\n",
       "  '',\n",
       "  '0.3',\n",
       "  '',\n",
       "  '0.4',\n",
       "  '',\n",
       "  'Figure 17.3 Open-loop response CAt−1xinit.',\n",
       "  '',\n",
       "  '0.7738942551160318',\n",
       "  '',\n",
       "  'julia> J_output = norm(y)^2',\n",
       "  '',\n",
       "  '3.7829986463323224',\n",
       "  '',\n",
       "  'julia> plot(1:T-1, vcat(u...), legend = false, xlabel=\"t\",',\n",
       "  '',\n",
       "  'ylabel= \"u_t\")',\n",
       "  '',\n",
       "  'julia> plot(1:T, vcat(y...), legend=false, xlabel = \"t\",',\n",
       "  '',\n",
       "  'ylabel = \"y_t\")',\n",
       "  '',\n",
       "  'Linear state feedback control. To finish the example we implement the state',\n",
       "  'feedback method in VMLS section 17.2.3. The plots in figure 17.5 reproduce VMLS',\n",
       "  'figure 17.7.',\n",
       "  '',\n",
       "  'julia> # Solve LQ problem with x_init = I, x_des = 0',\n",
       "  '',\n",
       "  'julia> rho = 1.0;',\n",
       "  '',\n",
       "  'julia> xsf, usf, ysf = lqr(A,B,C,eye(n),zeros(n,n),T,rho);',\n",
       "  '',\n",
       "  'julia> K = usf[1];',\n",
       "  '',\n",
       "  'julia> # Simulate over horizon 150',\n",
       "  '',\n",
       "  'julia> TT = 150;',\n",
       "  '',\n",
       "  'julia> Xsf = [x_init zeros(n,TT-1)];',\n",
       "  '',\n",
       "  'julia> for k=1:TT-1',\n",
       "  '',\n",
       "  'Xsf[:,k+1] = (A+B*K)*Xsf[:,k];',\n",
       "  '',\n",
       "  'end;',\n",
       "  '',\n",
       "  'julia> usf = K*Xsf[:, 1:TT-1];',\n",
       "  '',\n",
       "  'julia> ysf = C*Xsf;',\n",
       "  '',\n",
       "  'julia> # Also compute optimal LQ solution for rho = 1.0',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#subsection.17.2.3',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.17.7',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '17.2 Linear quadratic control 141',\n",
       "  '',\n",
       "  '0 25 50 75 100',\n",
       "  '',\n",
       "  '-0.2',\n",
       "  '',\n",
       "  '-0.1',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '0.1',\n",
       "  '',\n",
       "  '0.2',\n",
       "  '',\n",
       "  '0.3',\n",
       "  '',\n",
       "  't',\n",
       "  '',\n",
       "  'u_',\n",
       "  't',\n",
       "  '',\n",
       "  '0 25 50 75 100',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '0.1',\n",
       "  '',\n",
       "  '0.2',\n",
       "  '',\n",
       "  '0.3',\n",
       "  '',\n",
       "  '0.4',\n",
       "  '',\n",
       "  't',\n",
       "  '',\n",
       "  'y_',\n",
       "  't',\n",
       "  '',\n",
       "  'Figure 17.4 Optimal input and output for ρ = 0.2.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '142 17 Constrained least squares applications',\n",
       "  '',\n",
       "  '0 50 100 150',\n",
       "  '',\n",
       "  '-0.10',\n",
       "  '',\n",
       "  '-0.05',\n",
       "  '',\n",
       "  '0.00',\n",
       "  '',\n",
       "  '0.05',\n",
       "  '',\n",
       "  '0.10',\n",
       "  '',\n",
       "  't',\n",
       "  '',\n",
       "  'u_',\n",
       "  't',\n",
       "  '',\n",
       "  'Optimal',\n",
       "  'State feedback',\n",
       "  '',\n",
       "  '0 50 100 150',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '0.1',\n",
       "  '',\n",
       "  '0.2',\n",
       "  '',\n",
       "  '0.3',\n",
       "  '',\n",
       "  '0.4',\n",
       "  '',\n",
       "  't',\n",
       "  '',\n",
       "  'y_',\n",
       "  't',\n",
       "  '',\n",
       "  'Optimal',\n",
       "  'State feedback',\n",
       "  '',\n",
       "  'Figure 17.5 The blue curves are the solutions of the linear quadratic control',\n",
       "  'problem for ρ = 1. The red curves are the inputs and outputs that result',\n",
       "  'from the constant state feedback ut = Kxt.',\n",
       "  '',\n",
       "  '.',\n",
       "  '',\n",
       "  'julia> x, u, y = lqr(A,B,C,x_init,x_des,T,rho)',\n",
       "  '',\n",
       "  'julia> # Plot the two inputs',\n",
       "  '',\n",
       "  'julia> plot([vcat(u...); zeros(TT-T,1)],',\n",
       "  '',\n",
       "  'label=\"Optimal\", xlabel = \"t\", ylabel = \"u_t\")',\n",
       "  '',\n",
       "  'julia> plot!(usf\\', label = \"State feedback\")',\n",
       "  'julia> # Plot the two outputs',\n",
       "  '',\n",
       "  'julia> plot_sf_y = plot([vcat(y...); zeros(TT-T,1)],',\n",
       "  '',\n",
       "  'label=\"Optimal\", xlabel = \"t\", ylabel = \"y_t\")',\n",
       "  '',\n",
       "  'julia> plot!(ysf\\', label = \"State feedback\")',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '17.3 Linear quadratic state estimation 143',\n",
       "  '',\n",
       "  '17.3 Linear quadratic state estimation',\n",
       "  '',\n",
       "  'The code for the linear quadratic estimation method is very similar to the one for',\n",
       "  'linear quadratic control.',\n",
       "  '',\n",
       "  'function lqe(A,B,C,y,T,lambda)',\n",
       "  '',\n",
       "  'n = size(A,1)',\n",
       "  '',\n",
       "  'm = size(B,2)',\n",
       "  '',\n",
       "  'p = size(C,1)',\n",
       "  '',\n",
       "  'Atil = [ kron(eye(T), C) zeros(T*p, m*(T-1));',\n",
       "  '',\n",
       "  'zeros(m*(T-1), n*T) sqrt(lambda)*eye(m*(T-1)) ]',\n",
       "  '',\n",
       "  '# We assume y is a p x T array, so we vectorize it',\n",
       "  '',\n",
       "  'btil = [ vcat(y...) ; zeros((T-1)*m) ]',\n",
       "  '',\n",
       "  'Ctil = [ ([ kron(eye(T-1), A) zeros(n*(T-1), n) ] +',\n",
       "  '',\n",
       "  '[ zeros(n*(T-1), n) -eye(n*(T-1)) ]) kron(eye(T-1), B) ]',\n",
       "  '',\n",
       "  'dtil = zeros(n*(T-1))',\n",
       "  '',\n",
       "  'z = cls_solve(Atil, btil, Ctil, dtil)',\n",
       "  '',\n",
       "  'x = [ z[(i-1)*n+1:i*n] for i=1:T ]',\n",
       "  '',\n",
       "  'u = [ z[n*T+(i-1)*m+1 : n*T+i*m] for i=1:T-1 ]',\n",
       "  '',\n",
       "  'y = [ C*xt for xt in x ]',\n",
       "  '',\n",
       "  'return x, u, y',\n",
       "  '',\n",
       "  'end',\n",
       "  '',\n",
       "  'We use the system matrices in §17.3.1 of VMLS. The output measurement data',\n",
       "  'are read from an input file estimation_data.jl, which creates a 2 × 100 matrix',\n",
       "  'ymeas. We compute the solution for λ = 103, shown in the lower-left plot of VMLS',\n",
       "  'figure 17.8.',\n",
       "  '',\n",
       "  'julia> ymeas = lq_estimation_data();',\n",
       "  '',\n",
       "  'julia> A = [ eye(2) eye(2); zeros(2,2) eye(2) ];',\n",
       "  '',\n",
       "  'julia> B = [ zeros(2,2); eye(2) ];',\n",
       "  '',\n",
       "  'julia> C = [ eye(2) zeros(2,2) ];',\n",
       "  '',\n",
       "  'julia> n = size(A,1);',\n",
       "  '',\n",
       "  'julia> m = size(B,2);',\n",
       "  '',\n",
       "  'julia> p = size(C,1);',\n",
       "  '',\n",
       "  'julia> T = 100;',\n",
       "  '',\n",
       "  'julia> lambda = 1e3;',\n",
       "  '',\n",
       "  'julia> xest, uest, yest = lqe(A,B,C,y,T,lambda)',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> scatter(y[1,:], y[2,:], legend = false, axis = false)',\n",
       "  '',\n",
       "  'julia> plot!( [yt[1] for yt in yest], [yt[2] for yt in yest])',\n",
       "  '',\n",
       "  'The result can be seen in figure 17.6',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#subsection.17.3.1',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.17.8',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '144 17 Constrained least squares applications',\n",
       "  '',\n",
       "  'Figure 17.6 The circles show 100 noisy measurements in 2-D. The solid line',\n",
       "  'is the estimated trajectory Cx̂t for λ = 1000.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Chapter 18',\n",
       "  '',\n",
       "  'Nonlinear least squares',\n",
       "  '',\n",
       "  '18.1 Nonlinear equations and least squares',\n",
       "  '',\n",
       "  '18.2 Gauss–Newton algorithm',\n",
       "  '',\n",
       "  'Basic Gauss–Newton algorithm. Let’s first implement the basic Gauss–Newton',\n",
       "  'method (algorithm 18.1 in VMLS) in Julia. In Julia, you can pass a function as',\n",
       "  'an argument to another function, so we can pass f (the function) and also Df (the',\n",
       "  'derivative or Jacobian matrix) to our Gauss–Newton algorithm.',\n",
       "  '',\n",
       "  '1 function gauss_newton(f, Df, x1; kmax = 10)',\n",
       "  '',\n",
       "  '2 x = x1',\n",
       "  '',\n",
       "  '3 for k = 1:kmax',\n",
       "  '',\n",
       "  '4 x = x - Df(x) \\\\ f(x)',\n",
       "  '',\n",
       "  '5 end',\n",
       "  '',\n",
       "  '6 return x',\n",
       "  '',\n",
       "  '7 end',\n",
       "  '',\n",
       "  'Here we simply run the algorithm for a fixed number of iterations kmax, specified',\n",
       "  'by an optional keyword argument with default value 10. The code does not verify',\n",
       "  'whether the final x is actually a solution, and it will break down when Df(x(k)) has',\n",
       "  'linearly dependent columns. This very simple implementation is only for illustrative',\n",
       "  'purposes; the Levenberg–Marquardt algorithm described in the next section is',\n",
       "  'better in every way.',\n",
       "  '',\n",
       "  'Newton algorithm. The Gauss–Newton algorithm reduces to the Newton algo-',\n",
       "  'rithm when the function maps n-vectors to n-vectors, so the function above is also',\n",
       "  'an implementation of the Newton method for solving nonlinear equations. The',\n",
       "  'only difference with the following function is the stopping condition. In Newton’s',\n",
       "  'method one terminates when ‖f(x(k))‖ is sufficiently small.',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#algorithmctr.18.1',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '146 18 Nonlinear least squares',\n",
       "  '',\n",
       "  '1 function newton(f, Df, x1; kmax = 20, tol = 1e-6)',\n",
       "  '',\n",
       "  '2 x = x1',\n",
       "  '',\n",
       "  '3 fnorms = zeros(1,0)',\n",
       "  '',\n",
       "  '4 for k = 1:kmax',\n",
       "  '',\n",
       "  '5 fk = f(x)',\n",
       "  '',\n",
       "  '6 fnorms = [fnorms; norm(fk)]',\n",
       "  '',\n",
       "  '7 if norm(fk) < tol',\n",
       "  '',\n",
       "  '8 break',\n",
       "  '',\n",
       "  '9 end;',\n",
       "  '',\n",
       "  '10 x = x - Df(x) \\\\ fk',\n",
       "  '',\n",
       "  '11 end',\n",
       "  '',\n",
       "  '12 return x, fnorms',\n",
       "  '',\n",
       "  '13 end',\n",
       "  '',\n",
       "  'We added a second optional argument with the tolerance in the stopping condition',\n",
       "  'on line 7. The default value is 10−6. We also added a second output argument',\n",
       "  'fnorms, with the sequence ‖f(x(k))‖, so we can examine the convergence in the',\n",
       "  'following examples.',\n",
       "  '',\n",
       "  'Newton algorithm for n = 1. Our first example is a scalar nonlinear equation',\n",
       "  'f(x) = 0 with',\n",
       "  '',\n",
       "  'f(x) =',\n",
       "  'ex − e−x',\n",
       "  '',\n",
       "  'ex + e−x',\n",
       "  '(18.1)',\n",
       "  '',\n",
       "  '(VMLS figures 18.3 and 18.4).',\n",
       "  '',\n",
       "  'julia> f(x) = (exp(x)-exp(-x)) / (exp(x)+exp(-x));',\n",
       "  '',\n",
       "  'julia> Df(x) = 4 / (exp(x) + exp(-x))^2;',\n",
       "  '',\n",
       "  'We first try with x(1) = 0.95.',\n",
       "  '',\n",
       "  'julia> x, fnorms = newton(f,Df,0.95);',\n",
       "  '',\n",
       "  'julia> f(x)',\n",
       "  '',\n",
       "  '4.3451974324200454e-7',\n",
       "  '',\n",
       "  'julia> fnorms',\n",
       "  '',\n",
       "  '5-element Array{Float64,2}:',\n",
       "  '',\n",
       "  '0.7397830512740042',\n",
       "  '',\n",
       "  '0.5941663642651942',\n",
       "  '',\n",
       "  '0.23011124550034218',\n",
       "  '',\n",
       "  '0.00867002864500575',\n",
       "  '',\n",
       "  '4.3451974324200454e-7',\n",
       "  '',\n",
       "  'julia> Using Plots',\n",
       "  '',\n",
       "  'julia> plot(fnorms, shape=:circle, legend = false, xlabel = \"k\",',\n",
       "  '',\n",
       "  'ylabel = \"|f|\")',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.18.3',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.18.4',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '18.3 Levenberg–Marquardt algorithm 147',\n",
       "  '',\n",
       "  '1 2 3 4 5',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '0.2',\n",
       "  '',\n",
       "  '0.4',\n",
       "  '',\n",
       "  '0.6',\n",
       "  '',\n",
       "  'k',\n",
       "  '',\n",
       "  '|f|',\n",
       "  '',\n",
       "  'Figure 18.1 The first iterations in the Newton algorithm for solving f(x) = 0',\n",
       "  'for starting point x(1) = 0.95.',\n",
       "  '',\n",
       "  'The method converges very quickly, as can also be seen in figure 18.1. However it',\n",
       "  'does not converge for a slightly larger starting point x(1) = 1.15.',\n",
       "  '',\n",
       "  'julia> x, fnorms = newton(f,Df,1.15);',\n",
       "  '',\n",
       "  'julia> f(x)',\n",
       "  '',\n",
       "  'NaN',\n",
       "  '',\n",
       "  'julia> fnorms[1:5]',\n",
       "  '',\n",
       "  '5-element Array{Float64,2}:',\n",
       "  '',\n",
       "  '0.8177540779702877',\n",
       "  '',\n",
       "  '0.8664056534177534',\n",
       "  '',\n",
       "  '0.9735568532451108',\n",
       "  '',\n",
       "  '0.9999999999999906',\n",
       "  '',\n",
       "  'NaN',\n",
       "  '',\n",
       "  '18.3 Levenberg–Marquardt algorithm',\n",
       "  '',\n",
       "  'The Gauss–Newton algorithm can fail if the derivative matrix does not have in-',\n",
       "  'dependent columns. It also does not guarantee that ‖f(x(k))‖ decreases in each',\n",
       "  'iteration. Both of these shortcomings are addressed in the Levenberg–Marquardt',\n",
       "  'algorithm. Below is a Julia implementation of algorithm 18.3 in VMLS. This func-',\n",
       "  'tion is included in the Vmls package.',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#algorithmctr.18.3',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '148 18 Nonlinear least squares',\n",
       "  '',\n",
       "  '1 function levenberg_marquardt(f, Df, x1, lambda1; kmax=100, tol=1e-6)',\n",
       "  '',\n",
       "  '2 n = length(x1)',\n",
       "  '',\n",
       "  '3 x = x1',\n",
       "  '',\n",
       "  '4 lambda = lambda1',\n",
       "  '',\n",
       "  '5 objectives = zeros(0,1)',\n",
       "  '',\n",
       "  '6 residuals = zeros(0,1)',\n",
       "  '',\n",
       "  '7 for k = 1:kmax',\n",
       "  '',\n",
       "  '8 fk = f(x)',\n",
       "  '',\n",
       "  '9 Dfk = Df(x)',\n",
       "  '',\n",
       "  '10 objectives = [objectives; norm(fk)^2]',\n",
       "  '',\n",
       "  \"11 residuals = [residuals; norm(2*Dfk'*fk)]\",\n",
       "  \"12 if norm(2*Dfk'*fk) < tol\",\n",
       "  '13 break',\n",
       "  '',\n",
       "  '14 end;',\n",
       "  '',\n",
       "  '15 xt = x - [ Dfk; sqrt(lambda)*eye(n) ] \\\\ [ fk; zeros(n) ]',\n",
       "  '',\n",
       "  '16 if norm(f(xt)) < norm(fk)',\n",
       "  '',\n",
       "  '17 lambda = 0.8*lambda',\n",
       "  '',\n",
       "  '18 x = xt',\n",
       "  '',\n",
       "  '19 else',\n",
       "  '',\n",
       "  '20 lambda = 2.0*lambda',\n",
       "  '',\n",
       "  '21 end',\n",
       "  '',\n",
       "  '22 end',\n",
       "  '',\n",
       "  '23 return x, Dict([ (\"objectives\", objectives),',\n",
       "  '',\n",
       "  '24 (\"residuals\", residuals)])',\n",
       "  '',\n",
       "  '25 end',\n",
       "  '',\n",
       "  'Line 12 is the second stopping criterion suggested on page 393 of VMLS, and checks',\n",
       "  'whether the optimality condition (18.3) is approximately satisfied. The default',\n",
       "  'tolerance 10−6 can vary with the scale of the problem and the desired accuracy.',\n",
       "  'Keep in mind that the optimality condition (18.3) is a necessary condition and',\n",
       "  'does not guarantee that the solution minimizes the nonlinear least squares objective',\n",
       "  '‖f(x)‖2. The code limits the number of iterations to kmax, after which it is assumed',\n",
       "  'that the algorithm is failing to converge.',\n",
       "  '',\n",
       "  'The function returns a dictionary with information about the sequence of iter-',\n",
       "  'ates, including the value of ‖f(x(k)‖2 and ‖Df(x(k))T f(x(k))‖ at each iteration.',\n",
       "  '',\n",
       "  'Nonlinear equation. We apply the algorithm to the scalar function (18.1) with',\n",
       "  'the starting point x(1) = 1.15.',\n",
       "  '',\n",
       "  'julia> f(x) = (exp.(x) - exp.(-x)) / (exp.(x) + exp.(-x));',\n",
       "  '',\n",
       "  'julia> Df(x) = 4 ./ (exp.(x) + exp.(-x)).^2;',\n",
       "  '',\n",
       "  'julia> x, history = levenberg_marquardt(f, Df, [1.15], 1.0);',\n",
       "  '',\n",
       "  'julia> plot(sqrt.(history[\"objectives\"][1:10]), shape = :circle,',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.408',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#equation.18.1.3',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#equation.18.1.3',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '18.3 Levenberg–Marquardt algorithm 149',\n",
       "  '',\n",
       "  '2 4 6 8 10',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '0.2',\n",
       "  '',\n",
       "  '0.4',\n",
       "  '',\n",
       "  '0.6',\n",
       "  '',\n",
       "  '0.8',\n",
       "  '',\n",
       "  'k',\n",
       "  '',\n",
       "  '|f|',\n",
       "  '',\n",
       "  'Figure 18.2 Values of |f(x(k))| versus the iteration number k for',\n",
       "  'the Levenberg–Marquardt algorithm applied to f(x) = (exp(x) −',\n",
       "  'exp(−x))/(exp(x)+exp(−x)). The starting point is x(1) = 1.15 and λ(1) = 1.',\n",
       "  '',\n",
       "  'legend = false, xlabel = \"k\", ylabel = \"|f|\");',\n",
       "  '',\n",
       "  'The result is shown in figure 18.2.',\n",
       "  'Note that we defined x(1) as the array [1.15], and use dot-operations in the',\n",
       "  '',\n",
       "  'definitions of f and Df to ensure that these functions work with vector arguments.',\n",
       "  'This is important because Julia distinguishes between scalars and 1-vectors. If we',\n",
       "  'call the levenberg_marquardt function with a scalar argument x1, line 15 will',\n",
       "  'raise an error, because Julia does not accept subtractions of scalars and 1-vectors.',\n",
       "  '',\n",
       "  'Equilibrium prices. We solve a nonlinear equation f(p) = 0 with two variables,',\n",
       "  'where',\n",
       "  '',\n",
       "  'f(p) = exp(Es log p+ snom)− exp(Ed log p+ dnom). (18.2)',\n",
       "  '',\n",
       "  'Here exp and log are interpreted as element-wise vector operations. The problem',\n",
       "  'parameters are snom = (2.2, 0.3), dnom = (3.1, 2.2),',\n",
       "  '',\n",
       "  'Es =',\n",
       "  '',\n",
       "  '[',\n",
       "  '0.5 −0.3',\n",
       "  '−0.15 0.8',\n",
       "  '',\n",
       "  ']',\n",
       "  ', Ed =',\n",
       "  '',\n",
       "  '[',\n",
       "  '−0.5 0.2',\n",
       "  '',\n",
       "  '0 −0.5',\n",
       "  '',\n",
       "  ']',\n",
       "  '.',\n",
       "  '',\n",
       "  'julia> snom = [2.2, 0.3];',\n",
       "  '',\n",
       "  'julia> dnom = [3.1, 2.2];',\n",
       "  '',\n",
       "  'julia> Es = [0.5 -.3; -0.15 0.8];',\n",
       "  '',\n",
       "  'julia> Ed = [-0.5 0.2; -0.00 -0.5];',\n",
       "  '',\n",
       "  'julia> f(p) = exp.(Es * log.(p) + snom) - exp.(Ed * log.(p) + dnom);',\n",
       "  '',\n",
       "  'julia> function Df(p)',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '150 18 Nonlinear least squares',\n",
       "  '',\n",
       "  '2.5 5.0 7.5 10.0 12.5',\n",
       "  '',\n",
       "  '0',\n",
       "  '',\n",
       "  '25',\n",
       "  '',\n",
       "  '50',\n",
       "  '',\n",
       "  '75',\n",
       "  '',\n",
       "  '100',\n",
       "  '',\n",
       "  '125',\n",
       "  '',\n",
       "  'k',\n",
       "  '',\n",
       "  'O',\n",
       "  'bj',\n",
       "  '',\n",
       "  'ec',\n",
       "  'tiv',\n",
       "  '',\n",
       "  'e',\n",
       "  '',\n",
       "  'Figure 18.3 Cost function ‖f(p(k)‖2 versus iteration number k for the ex-',\n",
       "  'ample of equation (18.2).',\n",
       "  '',\n",
       "  'S = exp.(Es * log.(p) + snom);',\n",
       "  '',\n",
       "  'D = exp.(Ed * log.(p) + dnom);',\n",
       "  '',\n",
       "  'return [ S[1]*Es[1,1]/p[1] S[1]*Es[1,2]/p[2];',\n",
       "  '',\n",
       "  'S[2]*Es[2,1]/p[1] S[2]*Es[2,2]/p[2] ] -',\n",
       "  '',\n",
       "  '[ D[1]*Ed[1,1]/p[1] D[1]*Ed[1,2]/p[2];',\n",
       "  '',\n",
       "  'D[2]*Ed[2,1]/p[1] D[2]*Ed[2,2]/p[2] ];',\n",
       "  '',\n",
       "  'end;',\n",
       "  '',\n",
       "  'julia> p, history = levenberg_marquardt(f, Df, [3, 9], 1);',\n",
       "  '',\n",
       "  'julia> p',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '5.379958476145877',\n",
       "  '',\n",
       "  '4.996349602562754',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> plot(history[\"objectives\"], shape = :circle, legend =false,',\n",
       "  '',\n",
       "  'xlabel = \"k\", ylabel = \"Objective\")',\n",
       "  '',\n",
       "  'Figure 18.3 shows the plot of ‖f(p(k))‖2 versus iteration number k.',\n",
       "  '',\n",
       "  'Location from range measurements. The next example is the location from',\n",
       "  'range measurements problem on page 396 in VMLS. The positions of the m = 5',\n",
       "  'points ai are given as rows in a 5 × 2 matrix A. The measurements are given in a',\n",
       "  '5-vector rhos. To simplify the code for the functions f(x) and Df(x) we add a',\n",
       "  'function dist(x) that computes the vector of distances (‖x− a1‖, . . . , ‖x− am‖).',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.415',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '18.3 Levenberg–Marquardt algorithm 151',\n",
       "  '',\n",
       "  'The expression for the derivative is',\n",
       "  '',\n",
       "  'Df(x) =',\n",
       "  '',\n",
       "  '\\uf8ee\\uf8ef\\uf8ef\\uf8ef\\uf8ef\\uf8ef\\uf8f0',\n",
       "  'x1 − (a1)1',\n",
       "  '‖x− a1‖',\n",
       "  '',\n",
       "  'x2 − (a1)2',\n",
       "  '‖x− a1‖',\n",
       "  '',\n",
       "  '...',\n",
       "  '...',\n",
       "  '',\n",
       "  'x1 − (am)1',\n",
       "  '‖x− am‖',\n",
       "  '',\n",
       "  'x2 − (am))2',\n",
       "  '‖x− am‖',\n",
       "  '',\n",
       "  '\\uf8f9\\uf8fa\\uf8fa\\uf8fa\\uf8fa\\uf8fa\\uf8fb .',\n",
       "  '',\n",
       "  'This can be evaluated as the product of a diagonal matrix with diagonal entries',\n",
       "  '1/‖x− ai‖ and the 5× 2 matrix with i, j entry (x− ai)j .',\n",
       "  '',\n",
       "  'We run the Levenberg–Marquardt method for three starting points and λ(1) =',\n",
       "  '0.1. The plot is shown in figure 18.4.',\n",
       "  '',\n",
       "  'julia> # Five locations ai in a 5x2 matrix.',\n",
       "  '',\n",
       "  'julia> A = [ 1.8 2.5; 2.0 1.7; 1.5 1.5; 1.5 2.0; 2.5 1.5 ];',\n",
       "  '',\n",
       "  'julia> # Vector of measured distances to five locations.',\n",
       "  '',\n",
       "  'julia> rhos = [ 1.87288, 1.23950, 0.53672, 1.29273, 1.49353 ];',\n",
       "  '',\n",
       "  'julia> # dist(x) returns a 5-vector with the distances ||x-ai||.',\n",
       "  '',\n",
       "  'julia> dist(x) = sqrt.( (x[1] .- A[:,1]).^2 + (x[2] .- A[:,2]).^2 );',\n",
       "  '',\n",
       "  'julia> # f(x) returns the five residuals.',\n",
       "  '',\n",
       "  'julia> f(x) = dist(x) - rhos;',\n",
       "  '',\n",
       "  'julia> # Df(x) is the 5x2 derivative.',\n",
       "  '',\n",
       "  'julia> Df(x) = diagonal(1 ./ dist(x)) *',\n",
       "  '',\n",
       "  '[ (x[1] .- A[:,1]) (x[2] .- A[:,2]) ];',\n",
       "  '',\n",
       "  'julia> # Solve with starting point (1.8,3.5) and lambda = 0.1.',\n",
       "  '',\n",
       "  'julia> x1, history1 = levenberg_marquardt(f, Df, [1.8, 3.5], 0.1);',\n",
       "  '',\n",
       "  'julia> x1',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '1.1824859803827907',\n",
       "  '',\n",
       "  '0.8242289367900364',\n",
       "  '',\n",
       "  'julia> # Starting point (3.0,1.5).',\n",
       "  '',\n",
       "  'julia> x2, history2 = levenberg_marquardt(f, Df, [3.0, 1.5], 0.1);',\n",
       "  '',\n",
       "  'julia> x2',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '1.1824857942435818',\n",
       "  '',\n",
       "  '0.8242289466379732',\n",
       "  '',\n",
       "  'julia> # Starting point (2.2,3.5).',\n",
       "  '',\n",
       "  'julia> x3, history3 = levenberg_marquardt(f, Df, [2.2, 3.5], 0.1);',\n",
       "  '',\n",
       "  'julia> x3',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '2.9852664103617954',\n",
       "  '',\n",
       "  '2.1215768036188956',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '152 18 Nonlinear least squares',\n",
       "  '',\n",
       "  '2 4 6 8 10',\n",
       "  '0',\n",
       "  '',\n",
       "  '1',\n",
       "  '',\n",
       "  '2',\n",
       "  '',\n",
       "  '3',\n",
       "  '',\n",
       "  'k',\n",
       "  '',\n",
       "  'O',\n",
       "  'bj',\n",
       "  '',\n",
       "  'ec',\n",
       "  'tiv',\n",
       "  '',\n",
       "  'e',\n",
       "  '',\n",
       "  'Starting point 1',\n",
       "  'Starting point 2',\n",
       "  'Starting point 3',\n",
       "  '',\n",
       "  'Figure 18.4 Cost function ‖f(x(k))‖2 versus iteration number k for the three',\n",
       "  'starting points in the location from range measurements example.',\n",
       "  '',\n",
       "  'julia> plot(history1[\"objectives\"][1:10], shape = :circle,',\n",
       "  '',\n",
       "  'label = \"Starting point 1\")',\n",
       "  '',\n",
       "  'julia> plot!(history2[\"objectives\"][1:10], shape = :circle,',\n",
       "  '',\n",
       "  'label = \"Starting point 2\")',\n",
       "  '',\n",
       "  'julia> plot!(history3[\"objectives\"][1:10], shape = :circle,',\n",
       "  '',\n",
       "  'label = \"Starting point 3\")',\n",
       "  '',\n",
       "  'julia> plot!(xlabel = \"k\", ylabel = \"Objective\")',\n",
       "  '',\n",
       "  '18.4 Nonlinear model fitting',\n",
       "  '',\n",
       "  'Example. We fit a model',\n",
       "  '',\n",
       "  'f̂(x; θ) = θ1e',\n",
       "  'θ2x cos(θ3x+ θ4)',\n",
       "  '',\n",
       "  'to N = 60 data points. We first generate the data.',\n",
       "  '',\n",
       "  'julia> # Use these parameters to generate data.',\n",
       "  '',\n",
       "  'julia> theta_ex = [1, -0.2, 2*pi/5, pi/3];',\n",
       "  '',\n",
       "  'julia> # Choose 60 points x between 0 and 20.',\n",
       "  '',\n",
       "  'julia> M = 30;',\n",
       "  '',\n",
       "  'julia> xd = [5*rand(M); 5 .+ 15*rand(M)];',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '18.4 Nonlinear model fitting 153',\n",
       "  '',\n",
       "  'julia> # Evaluate function at these points.',\n",
       "  '',\n",
       "  'julia> yd = theta_ex[1] * exp.(theta_ex[2]*xd) .*',\n",
       "  '',\n",
       "  'cos.(theta_ex[3] * xd .+ theta_ex[4])',\n",
       "  '',\n",
       "  'julia> # Create a random perturbation of yd.',\n",
       "  '',\n",
       "  'julia> N = length(xd);',\n",
       "  '',\n",
       "  'julia> yd = yd .* (1 .+ 0.2*randn(N)) .+ 0.015 * randn(N);',\n",
       "  '',\n",
       "  'julia> # Plot data points.',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> scatter(xd, yd, legend=false)',\n",
       "  '',\n",
       "  'The 60 points are shown in figure 18.5. We now run our Levenberg–Marquardt',\n",
       "  'code with starting point θ(1) = (1, 0, 1, 0) and λ(1) = 1. The fitted model is shown',\n",
       "  'in figure 18.5.',\n",
       "  '',\n",
       "  'julia> f(theta) = theta[1] * exp.(theta[2]*xd) .*',\n",
       "  '',\n",
       "  'cos.(theta[3] * xd .+ theta[4]) - yd;',\n",
       "  '',\n",
       "  'julia> Df(theta) = hcat(',\n",
       "  '',\n",
       "  'exp.(theta[2]*xd) .* cos.(theta[3] * xd .+ theta[4]),',\n",
       "  '',\n",
       "  'theta[1] * ( xd .* exp.(theta[2]*xd) .*',\n",
       "  '',\n",
       "  'cos.(theta[3] * xd .+ theta[4])),',\n",
       "  '',\n",
       "  '-theta[1] * ( exp.(theta[2]*xd) .* xd .*',\n",
       "  '',\n",
       "  'sin.(theta[3] * xd .+ theta[4])),',\n",
       "  '',\n",
       "  '-theta[1] * ( exp.(theta[2]*xd) .*',\n",
       "  '',\n",
       "  'sin.(theta[3] * xd .+ theta[4])) );',\n",
       "  '',\n",
       "  'julia> theta1 = [1, 0, 1, 0];',\n",
       "  '',\n",
       "  'julia> theta, history = levenberg_marquardt(f, Df, theta1, 1.0)',\n",
       "  '',\n",
       "  'julia> theta',\n",
       "  '',\n",
       "  '4-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '1.0065969737811806',\n",
       "  '',\n",
       "  '-0.23115179954434736',\n",
       "  '',\n",
       "  '1.2697087881931268',\n",
       "  '',\n",
       "  '1.0133243392186635',\n",
       "  '',\n",
       "  'julia> # Plot the fitted model.',\n",
       "  '',\n",
       "  'julia> x = linspace(0, 20, 500);',\n",
       "  '',\n",
       "  'julia> y=theta[1]*exp.(theta[2]*x) .* cos.(theta[3]*x .+ theta[4]);',\n",
       "  '',\n",
       "  'julia> plot!(x, y, legend = false);',\n",
       "  '',\n",
       "  'Orthogonal distance regression. In figure 18.14 of VMLS we use orthogonal dis-',\n",
       "  'tance regression to fit a cubic polynomial',\n",
       "  '',\n",
       "  'f̂(x; θ) = θ1 + θ2x+ θ3x',\n",
       "  '2 + θ4x',\n",
       "  '',\n",
       "  '3',\n",
       "  '',\n",
       "  'to N = 25 data points.',\n",
       "  'We first read in the data and compute the standard least squares fit.',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.18.14',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '154 18 Nonlinear least squares',\n",
       "  '',\n",
       "  '0 5 10 15 20',\n",
       "  '',\n",
       "  '-0.50',\n",
       "  '',\n",
       "  '-0.25',\n",
       "  '',\n",
       "  '0.00',\n",
       "  '',\n",
       "  '0.25',\n",
       "  '',\n",
       "  '0.50',\n",
       "  '',\n",
       "  'Figure 18.5 Least squares fit of a function f̂(x; θ) = θ1e',\n",
       "  'θ2x cos(θ3x+ θ4) to',\n",
       "  '',\n",
       "  'N = 60 points (x(i), y(i)).',\n",
       "  '',\n",
       "  'julia> xd, yd = orth_dist_reg_data(); # 2 vectors of length N = 25',\n",
       "  '',\n",
       "  'julia> N = length(xd);',\n",
       "  '',\n",
       "  'julia> p = 4;',\n",
       "  '',\n",
       "  'julia> theta_ls = vandermonde(xd, p) \\\\ yd;',\n",
       "  '',\n",
       "  'The nonlinear least squares formulation on page 400 of VMLS has p+N variables',\n",
       "  'θ1, . . . , θp, u',\n",
       "  '',\n",
       "  '(1), . . . , u(N). We will store them in that order in the nonlinear least',\n",
       "  'squares vector variable. The objective is to minimize the squared norm of the',\n",
       "  '2N -vector \\uf8ee\\uf8ef\\uf8ef\\uf8ef\\uf8ef\\uf8ef\\uf8ef\\uf8ef\\uf8ef\\uf8f0',\n",
       "  '',\n",
       "  'f̂(u(1); θ)− y(1)',\n",
       "  '...',\n",
       "  '',\n",
       "  'f̂(u(N); θ)− y(N)',\n",
       "  'u(1) − x(1)',\n",
       "  '',\n",
       "  '...',\n",
       "  'u(N) − x(N)',\n",
       "  '',\n",
       "  '\\uf8f9\\uf8fa\\uf8fa\\uf8fa\\uf8fa\\uf8fa\\uf8fa\\uf8fa\\uf8fa\\uf8fb',\n",
       "  '',\n",
       "  'function f(x)',\n",
       "  '',\n",
       "  'theta = x[1:p];',\n",
       "  '',\n",
       "  'u = x[p+1:end];',\n",
       "  '',\n",
       "  'f1 = vandermonde(u,p)*theta - yd',\n",
       "  '',\n",
       "  'f2 = u - xd',\n",
       "  '',\n",
       "  'return [f1; f2]',\n",
       "  '',\n",
       "  'end;',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.18.14',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '18.5 Nonlinear least squares classification 155',\n",
       "  '',\n",
       "  'function Df(x)',\n",
       "  '',\n",
       "  'theta = x[1:p]',\n",
       "  '',\n",
       "  'u = x[p+1:end]',\n",
       "  '',\n",
       "  'D11 = vandermonde(u,p)',\n",
       "  '',\n",
       "  'D12 = diagonal(theta[2] .+ 2*theta[3]*u .+ 3*theta[4]*u.^2)',\n",
       "  '',\n",
       "  'D21 = zeros(N,p)',\n",
       "  '',\n",
       "  'D22 = eye(N)',\n",
       "  '',\n",
       "  'return [ D11 D12; D21 D22]',\n",
       "  '',\n",
       "  'end',\n",
       "  '',\n",
       "  'We now call levenberg_marquardt with these two functions. A natural choice for',\n",
       "  'the initial point is to use the least squares solution for the variables θ and the data',\n",
       "  'points x(i) for the variables u(i). We use λ(1) = 0.01.',\n",
       "  '',\n",
       "  'julia> sol, hist = levenberg_marquardt(f, Df, [theta_ls; xd], 0.01);',\n",
       "  '',\n",
       "  'julia> theta_od = sol[1:p];',\n",
       "  '',\n",
       "  'Figure 18.6 shows the two fitted polynomials.',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> scatter(xd,yd, label=\"\", legend = :topleft);',\n",
       "  '',\n",
       "  'julia> x = linspace(minimum(xd), maximum(xd), 500);',\n",
       "  '',\n",
       "  'julia> y_ls = vandermonde(x, p) * theta_ls;',\n",
       "  '',\n",
       "  'julia> y_od = vandermonde(x, p) * theta_od;',\n",
       "  '',\n",
       "  'julia> plot!(x, y_ls, label = \"LS\");',\n",
       "  '',\n",
       "  'julia> plot!(x, y_od, label = \"Orth. dist.\");',\n",
       "  '',\n",
       "  '18.5 Nonlinear least squares classification',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '156 18 Nonlinear least squares',\n",
       "  '',\n",
       "  '0.2 0.4 0.6 0.8 1.0',\n",
       "  '',\n",
       "  '-0.2',\n",
       "  '',\n",
       "  '-0.1',\n",
       "  '',\n",
       "  '0.0',\n",
       "  '',\n",
       "  '0.1',\n",
       "  '',\n",
       "  '0.2',\n",
       "  '',\n",
       "  'LS',\n",
       "  'Orth. dist.',\n",
       "  '',\n",
       "  'Figure 18.6 Least squares and orthogonal distance regression fit of a cubic',\n",
       "  'polynomial to 25 data points.',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Chapter 19',\n",
       "  '',\n",
       "  'Constrained nonlinear least',\n",
       "  'squares',\n",
       "  '',\n",
       "  '19.1 Constrained nonlinear least squares',\n",
       "  '',\n",
       "  '19.2 Penalty algorithm',\n",
       "  '',\n",
       "  'Let’s implement the penalty algorithm (algorithm 19.1 in VMLS).',\n",
       "  '',\n",
       "  '1 function penalty_method(f, Df, g, Dg, x1, lambda1; kmax = 100,',\n",
       "  '',\n",
       "  '2 feas_tol = 1e-4, oc_tol = 1e-4)',\n",
       "  '',\n",
       "  '3 x = x1',\n",
       "  '',\n",
       "  '4 mu = 1.0',\n",
       "  '',\n",
       "  '5 feas_res = [norm(g(x))]',\n",
       "  '',\n",
       "  \"6 oc_res = [norm(2*Df(x)'*f(x) + 2*mu*Dg(x)'*g(x))]\",\n",
       "  '7 lm_iters = zeros(Int64,0,1);',\n",
       "  '',\n",
       "  '8 for k=1:kmax',\n",
       "  '',\n",
       "  '9 F(x) = [f(x); sqrt(mu)*g(x)]',\n",
       "  '',\n",
       "  '10 DF(x) = [Df(x); sqrt(mu)*Dg(x)]',\n",
       "  '',\n",
       "  '11 x, hist = levenberg_marquardt(F,DF,x,lambda1,tol=oc_tol)',\n",
       "  '',\n",
       "  '12 feas_res = [feas_res; norm(g(x))]',\n",
       "  '',\n",
       "  '13 oc_res = [oc_res; hist[\"residuals\"][end]]',\n",
       "  '',\n",
       "  '14 lm_iters = [lm_iters; length(hist[\"residuals\"])]',\n",
       "  '',\n",
       "  '15 if norm(g(x)) < feas_tol',\n",
       "  '',\n",
       "  '16 break',\n",
       "  '',\n",
       "  '17 end',\n",
       "  '',\n",
       "  '18 mu = 2*mu',\n",
       "  '',\n",
       "  '19 end',\n",
       "  '',\n",
       "  '20 return x, Dict([ (\"lm_iterations\", lm_iters),',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#algorithmctr.19.1',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '158 19 Constrained nonlinear least squares',\n",
       "  '',\n",
       "  '21 (\"feas_res\", feas_res), (\"oc_res\", oc_res) ])',\n",
       "  '',\n",
       "  '22 end',\n",
       "  '',\n",
       "  'On line 11 we call the function levenberg_marquardt of the previous chapter to',\n",
       "  'minimize ‖F (x)‖2 where',\n",
       "  '',\n",
       "  'F (x) =',\n",
       "  '',\n",
       "  '[',\n",
       "  'f(x)√',\n",
       "  'µg(x)',\n",
       "  '',\n",
       "  ']',\n",
       "  '.',\n",
       "  '',\n",
       "  'We evaluate two residuals. The “feasibility” residual ‖g(x(k))‖ is the error in the',\n",
       "  'constraint g(x) = 0. The “optimality condition” residual is defined as',\n",
       "  '',\n",
       "  '‖2Df(x(k))T f(x(k)) + 2Dg(x(k))T z(k)‖',\n",
       "  '',\n",
       "  'where z(k) = 2µ(k−1)g(x(k)) (and we take µ(0) = µ(1)). On line 13, we obtain',\n",
       "  'the optimality condition residual as the last residual in the Levenberg–Marquardt',\n",
       "  'method. On line 20 we return the final x, and a dictionary containing the two se-',\n",
       "  'quences of residuals and the number of iterations used in each call to the Levenberg–',\n",
       "  'Marquardt algorithm.',\n",
       "  '',\n",
       "  'Example. We apply the method to a problem with two variables',\n",
       "  '',\n",
       "  'f(x1, x2) =',\n",
       "  '',\n",
       "  '[',\n",
       "  'x1 + exp(−x2)',\n",
       "  'x21 + 2x2 + 1',\n",
       "  '',\n",
       "  ']',\n",
       "  ', g(x1, x2) = x',\n",
       "  '',\n",
       "  '2',\n",
       "  '1 + x',\n",
       "  '',\n",
       "  '3',\n",
       "  '1 + x2 + x',\n",
       "  '',\n",
       "  '2',\n",
       "  '2.',\n",
       "  '',\n",
       "  'julia> f(x) = [ x[1] + exp(-x[2]), x[1]^2 + 2*x[2] + 1 ];',\n",
       "  '',\n",
       "  'julia> Df(x) = [ 1.0 - exp(-x[2]); 2*x[1] 2 ];',\n",
       "  '',\n",
       "  'julia> g(x) = [ x[1] + x[1]^3 + x[2] + x[2]^2 ];',\n",
       "  '',\n",
       "  'julia> Dg(x) = [ 1 + 3*x[1]^2 1 + 2*x[2] ];',\n",
       "  '',\n",
       "  'julia> x, hist = penalty_method(f, Df, g, Dg, [0.5, -0.5], 1.0);',\n",
       "  '',\n",
       "  'julia> x',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '-3.334955140841332e-5',\n",
       "  '',\n",
       "  '-2.7682497163944097e-5',\n",
       "  '',\n",
       "  'The following lines create a staircase plot with the residuals versus the cumulative',\n",
       "  'number of Levenberg–Marquardt iterations as in VMLS figure 19.4. The result is',\n",
       "  'in figure 19.1.',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> cum_lm_iters = cumsum(hist[\"lm_iterations\"], dims=1);',\n",
       "  '',\n",
       "  'julia> itr = vcat([0], [[i; i] for i in cum_lm_iters]...)',\n",
       "  '',\n",
       "  'julia> feas_res = vcat([',\n",
       "  '',\n",
       "  '[r;r] for r in hist[\"feas_res\"][1:end-1]]...,',\n",
       "  '',\n",
       "  'hist[\"feas_res\"][end]);',\n",
       "  '',\n",
       "  'julia> oc_res = vcat([',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.19.4',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '19.3 Augmented Lagrangian algorithm 159',\n",
       "  '',\n",
       "  '0 20 40 60 80 100 120',\n",
       "  '',\n",
       "  '10-5 ',\n",
       "  '',\n",
       "  '10-4 ',\n",
       "  '',\n",
       "  '10-3 ',\n",
       "  '',\n",
       "  '10-2 ',\n",
       "  '',\n",
       "  '10-1 ',\n",
       "  '',\n",
       "  '100 ',\n",
       "  '',\n",
       "  '101 ',\n",
       "  '',\n",
       "  'Cumulative Levenberg-Marquardt iterations',\n",
       "  '',\n",
       "  'R',\n",
       "  'es',\n",
       "  '',\n",
       "  'id',\n",
       "  'ua',\n",
       "  '',\n",
       "  'l',\n",
       "  '',\n",
       "  'Feasibility',\n",
       "  'Opt. cond.',\n",
       "  '',\n",
       "  'Figure 19.1 Feasibility and optimality condition errors versus the cumulative',\n",
       "  'number of Levenberg–Marquardt iterations in the penalty algorithm.',\n",
       "  '',\n",
       "  '[r;r] for r in hist[\"oc_res\"][1:end-1]]...,',\n",
       "  '',\n",
       "  'hist[\"oc_res\"][end]);',\n",
       "  '',\n",
       "  'julia> plot(itr, feas_res, shape=:circle, label = \"Feasibility\")',\n",
       "  '',\n",
       "  'julia> plot!(itr, oc_res, shape=:circle, label = \"Opt. cond.\")',\n",
       "  '',\n",
       "  'julia> plot!(yscale = :log10,',\n",
       "  '',\n",
       "  'xlabel = \"Cumulative Levenberg--Marquardt iterations\",',\n",
       "  '',\n",
       "  'ylabel = \"Residual\")',\n",
       "  '',\n",
       "  '19.3 Augmented Lagrangian algorithm',\n",
       "  '',\n",
       "  '1 function aug_lag_method(f, Df, g, Dg, x1, lambda1; kmax = 100,',\n",
       "  '',\n",
       "  '2 feas_tol = 1e-4, oc_tol = 1e-4)',\n",
       "  '',\n",
       "  '3 x = x1',\n",
       "  '',\n",
       "  '4 z = zeros(length(g(x)))',\n",
       "  '',\n",
       "  '5 mu = 1.0',\n",
       "  '',\n",
       "  '6 feas_res = [norm(g(x))]',\n",
       "  '',\n",
       "  \"7 oc_res = [norm(2*Df(x)'*f(x) + 2*mu*Dg(x)'*z)]\",\n",
       "  '8 lm_iters = zeros(Int64,0,1);',\n",
       "  '',\n",
       "  '9 for k=1:kmax',\n",
       "  '',\n",
       "  '10 F(x) = [f(x); sqrt(mu)*(g(x) + z/(2*mu))]',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '160 19 Constrained nonlinear least squares',\n",
       "  '',\n",
       "  '11 DF(x) = [Df(x); sqrt(mu)*Dg(x)]',\n",
       "  '',\n",
       "  '12 x, hist = levenberg_marquardt(F, DF, x, lambda1, tol=oc_tol)',\n",
       "  '',\n",
       "  '13 z = z + 2*mu*g(x)',\n",
       "  '',\n",
       "  '14 feas_res = [feas_res; norm(g(x))]',\n",
       "  '',\n",
       "  '15 oc_res = [oc_res; hist[\"residuals\"][end]]',\n",
       "  '',\n",
       "  '16 lm_iters = [lm_iters; length(hist[\"residuals\"])]',\n",
       "  '',\n",
       "  '17 if norm(g(x)) < feas_tol',\n",
       "  '',\n",
       "  '18 break',\n",
       "  '',\n",
       "  '19 end',\n",
       "  '',\n",
       "  '20 mu = (norm(g(x)) < 0.25*feas_res[end-1]) ? mu : 2*mu',\n",
       "  '',\n",
       "  '21 end',\n",
       "  '',\n",
       "  '22 return x, z, Dict([ (\"lm_iterations\", lm_iters),',\n",
       "  '',\n",
       "  '23 (\"feas_res\", feas_res), (\"oc_res\", oc_res)])',\n",
       "  '',\n",
       "  '24 end',\n",
       "  '',\n",
       "  'Here the call to the Levenberg–Marquardt algorithm on line 12 is to minimizes',\n",
       "  '‖F (x)‖2 where',\n",
       "  '',\n",
       "  'F (x) =',\n",
       "  '',\n",
       "  '[',\n",
       "  'f(x)√',\n",
       "  '',\n",
       "  'µ(k)(g(x) + z(k)/(2µ(k)))',\n",
       "  '',\n",
       "  ']',\n",
       "  '.',\n",
       "  '',\n",
       "  'We again record the feasibility residuals ‖g(x(k)‖ and the optimality conditions',\n",
       "  'residuals',\n",
       "  '',\n",
       "  '‖2Df(x(k))T f(x(k)) + 2Dg(x(k))T z(k)‖,',\n",
       "  '',\n",
       "  'and return them in a dictionary.',\n",
       "  '',\n",
       "  'Example. We continue the small example.',\n",
       "  '',\n",
       "  'julia> x, z, hist = aug_lag_method(f, Df, g, Dg, [0.5, -0.5], 1.0);',\n",
       "  '',\n",
       "  'julia> x',\n",
       "  '',\n",
       "  '2-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '-1.8646614856169702e-5',\n",
       "  '',\n",
       "  '-1.5008567819930016e-5',\n",
       "  '',\n",
       "  'julia> z',\n",
       "  '',\n",
       "  '1-element Array{Float64,1}:',\n",
       "  '',\n",
       "  '-1.9999581273499105',\n",
       "  '',\n",
       "  'The following code shows the convergence as in VMLS figure 19.4. The plot is',\n",
       "  'given in figure 19.2.',\n",
       "  '',\n",
       "  'julia> using Plots',\n",
       "  '',\n",
       "  'julia> cum_lm_iters = cumsum(hist[\"lm_iterations\"],dims=1);',\n",
       "  '',\n",
       "  'julia> itr = vcat([0], [[i; i] for i in cum_lm_iters]...)',\n",
       "  '',\n",
       "  'julia> feas_res = vcat([',\n",
       "  '',\n",
       "  '[r;r] for r in hist[\"feas_res\"][1:end-1]]...,',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.19.4',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '19.4 Nonlinear control 161',\n",
       "  '',\n",
       "  '0 25 50 75 100',\n",
       "  '',\n",
       "  '10-4 ',\n",
       "  '',\n",
       "  '10-3 ',\n",
       "  '',\n",
       "  '10-2 ',\n",
       "  '',\n",
       "  '10-1 ',\n",
       "  '',\n",
       "  '100 ',\n",
       "  '',\n",
       "  '101 ',\n",
       "  '',\n",
       "  'Cumulative Levenberg-Marquardt iterations',\n",
       "  '',\n",
       "  'R',\n",
       "  'es',\n",
       "  '',\n",
       "  'id',\n",
       "  'ua',\n",
       "  '',\n",
       "  'l',\n",
       "  '',\n",
       "  'Feasibility',\n",
       "  'Opt. cond.',\n",
       "  '',\n",
       "  'Figure 19.2 Feasibility and optimality condition errors versus the cumulative',\n",
       "  'number of Levenberg–Marquardt iterations in the augmented Lagrangian',\n",
       "  'algorithm.',\n",
       "  '',\n",
       "  'hist[\"feas_res\"][end]);',\n",
       "  '',\n",
       "  'julia> oc_res = vcat([',\n",
       "  '',\n",
       "  '[r;r] for r in hist[\"oc_res\"][1:end-1]]...,',\n",
       "  '',\n",
       "  'hist[\"oc_res\"][end]);',\n",
       "  '',\n",
       "  'julia> plot(itr, feas_res, shape=:circle, , label = \"Feasibility\")',\n",
       "  '',\n",
       "  'julia> plot!(itr, oc_res, shape=:circle, label = \"Opt. cond.\")',\n",
       "  '',\n",
       "  'julia> plot!(yscale = :log10,',\n",
       "  '',\n",
       "  'xlabel = \"Cumulative Levenberg-Marquardt iterations\",',\n",
       "  '',\n",
       "  'ylabel = \"Residual\")',\n",
       "  '',\n",
       "  '19.4 Nonlinear control',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '162 19 Constrained nonlinear least squares',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Appendices',\n",
       "  '',\n",
       "  ''],\n",
       " 'Empty',\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'Appendix A',\n",
       "  '',\n",
       "  'The VMLS package',\n",
       "  '',\n",
       "  'After installing the VMLS package as described on page ix, you can use it by typing',\n",
       "  'using VMLS at the Julia prompt. Typing ? followed by a function name gives a',\n",
       "  'short description of the function.',\n",
       "  '',\n",
       "  'The VMLS package includes three types of functions: simple utility functions',\n",
       "  'that match the VMLS notation or are simpler to use than the corresponding Julia',\n",
       "  'functions, implementations of some algorithms in VMLS, and functions that gen-',\n",
       "  'erate data for some of the examples. The algorithm implementations are meant',\n",
       "  'for use in the examples of this companion and for exercises in VMLS. They are',\n",
       "  'not optimized for efficiency or robustness, and do not perform any error checking',\n",
       "  'of the input arguments.',\n",
       "  '',\n",
       "  'A.1 Utility functions',\n",
       "  '',\n",
       "  'Vector utility functions.',\n",
       "  '',\n",
       "  'avg(x). Returns the average of the elements of a vector or matrix (page 20).',\n",
       "  '',\n",
       "  'rms(x). Returns the RMS value of the elements of a vector or matrix (page 25).',\n",
       "  '',\n",
       "  'stdev(x). Returns the standard deviation of the elements of a vector or matrix',\n",
       "  '(page 28).',\n",
       "  '',\n",
       "  'ang(x,y). Returns the angle in radians between non-zero vectors (page 29).',\n",
       "  '',\n",
       "  'correl_coeff(x,y). Returns the correlation coefficient between non-constant',\n",
       "  'vectors (page 30).',\n",
       "  '',\n",
       "  'Matrix utility functions.',\n",
       "  '',\n",
       "  'eye(n). Returns an n× n identity matrix (page 50).',\n",
       "  '',\n",
       "  'diagonal(x). Returns a diagonal matrix with the entries of the vector x on its',\n",
       "  'diagonal (page 52).',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '166 A The VMLS package',\n",
       "  '',\n",
       "  'speye(n). Returns an n× n sparse identity matrix (page 54).',\n",
       "  '',\n",
       "  'spdiagonal(x). Returns a sparse diagonal matrix with the entries of the vector',\n",
       "  'x on its diagonal (page 54).',\n",
       "  '',\n",
       "  'vandermonde(t,n). Returns the Vandermonde matrix with n columns and ith',\n",
       "  'column ti−1 (page 58).',\n",
       "  '',\n",
       "  'toeplitz(a,n). Returns the Toeplitz matrix with n columns and the vector a in',\n",
       "  'the leading positions of the first column (page 65).',\n",
       "  '',\n",
       "  'Range function.',\n",
       "  '',\n",
       "  'linspace(a,b,n). Returns a vector with n equally spaced numbers between a',\n",
       "  'and b (page 90).',\n",
       "  '',\n",
       "  'Utility functions for classification.',\n",
       "  '',\n",
       "  'confusion_matrix(y,yhat,K=2). Returns the confusion matrix for a data vector',\n",
       "  'y and the vector of predictions ŷ. If K = 2, the vectors y and ŷ are Boolean.',\n",
       "  'If K > 2, they contain integers in {1, . . . ,K} (pages 113, 115).',\n",
       "  '',\n",
       "  'row_argmax(X). If X is an m × n matrix, returns an m-vector with ith element',\n",
       "  'argmaxj Xij (page 116).',\n",
       "  '',\n",
       "  'one_hot(x,K). Returns the one-hot encoding of the vector x, which must have',\n",
       "  'elements in {1, . . . ,K}. The one-hot encoding of an n-vector x is the n×K',\n",
       "  'matrix X with Xij = 1 if xi = j and Xij = 0 otherwise (page 117).',\n",
       "  '',\n",
       "  'A.2 Algorithms',\n",
       "  '',\n",
       "  'k_means(X,k;maxiters=100,tol=1e-5). Applies the k-means algorithm for k',\n",
       "  'clusters to the vectors stored in X. The argument X is a one-dimensional',\n",
       "  'array of N n-vectors, or an n × N -matrix. The function returns a tuple',\n",
       "  'with two elements. The first output argument is an array of N integers in',\n",
       "  '{1, . . . , k} with the cluster assignments for the N data points. The second',\n",
       "  'output argument is an array of k n-vectors with the k cluster representatives',\n",
       "  '(page 34).',\n",
       "  '',\n",
       "  'gram_schmidt(a;tol=1e-10). Applies the Gram–Schmidt algorithm to the vec-',\n",
       "  'tor stored in the array a and returns the result as an array of vectors (page 41).',\n",
       "  '',\n",
       "  'mols_solve(As,Bs,lambdas). Returns the solution of the multi-objective least',\n",
       "  'squares problem with coefficient matrices in the array As, right-hand side',\n",
       "  'vectors in the array bs, and weights in the array lambdas (page 121).',\n",
       "  '',\n",
       "  'cls_solve(A,b,C,d). Returns the solution of the constrained least squares prob-',\n",
       "  'lem with coefficient matrices A and C, and right-hand side vectors or matrices',\n",
       "  'b and d (page 132).',\n",
       "  '',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  'A.3 Data sets 167',\n",
       "  '',\n",
       "  'levenberg_marquardt(f,Df,x1,lambda1;kmax=100,tol=1e-6). Applies the Levenberg–',\n",
       "  'Marquardt algorithm to the function defined in f and Df, with starting point',\n",
       "  'x(1) and initial regularization parameter λ(1). The function returns the final',\n",
       "  'iterate x and a dictionary with the convergence history (page 147).',\n",
       "  '',\n",
       "  'aug_lag_method(f,Df,g,Dg,x1,lambda1;kmax=100,feas_tol=1e-4,oc_tol=1e-4).',\n",
       "  'Applies the augmented Lagrangian method to the constrained nonlinear least',\n",
       "  'squares problem defined by f, Df, g, Dg, with starting point x(1). The sub-',\n",
       "  'problems are solved using the Levenberg–Marquardt method with initial reg-',\n",
       "  'ularization parameter λ(1). Returns the final iterate x, multiplier z, and a',\n",
       "  'dictionary with the convergence history (page 159).',\n",
       "  '',\n",
       "  'A.3 Data sets',\n",
       "  '',\n",
       "  'house_sales_data(). Returns a dictionary D with the Sacramento house sales',\n",
       "  'data used in section 2.3 and chapter 13 of VMLS. The 6 items in the dictionary',\n",
       "  'are vectors of length 774, with data for 774 house sales.',\n",
       "  '',\n",
       "  'D[\"price\"]: selling price in 1000 dollars',\n",
       "  'D[\"area\"]: area in 1000 square feet',\n",
       "  'D[\"beds\"]: number of bedrooms',\n",
       "  'D[\"baths\"]: number of bathrooms',\n",
       "  'D[\"condo\"]: 1 if a condo, 0 otherwise',\n",
       "  'D[\"location\"]: an integer between 1 and 4 indicating the location.',\n",
       "  '',\n",
       "  'population_data(). Returns a dictionary D with the US population data used in',\n",
       "  'section 9.2 of VMLS. The items in the dictionary are three vectors of length',\n",
       "  '100.',\n",
       "  '',\n",
       "  'D[\"population\"]: 2010 population in millions for ages 0,. . . ,99',\n",
       "  'D[\"birth_rate\"]: birth rate',\n",
       "  'D[\"death_rate\"]: death rate.',\n",
       "  '',\n",
       "  'petroleum_consumption_data(). Returns a 34-vector with the world annual',\n",
       "  'petroleum consumption between 1980 and 2013, in thousand barrels/day (dis-',\n",
       "  'cussed on page 252 in VMLS).',\n",
       "  '',\n",
       "  'vehicle_miles_data(). Returns a 15 matrix with the vehicle miles traveled in',\n",
       "  'the US (in millions), per month, for the years 2000, . . . , 2014 (discussed on',\n",
       "  'page 252 in VMLS).',\n",
       "  '',\n",
       "  'temperature_data(). Returns a vector of length 774 = 31 · 24 with the hourly',\n",
       "  'temperature at LAX in May 2016 (discussed on pages 259 and 266 in VMLS).',\n",
       "  '',\n",
       "  'iris_data(). Returns a dictionary D with the Iris flower data set, discussed in',\n",
       "  'sections 14.2.1 and 14.3.2 of VMLS. The items in the dictionary are:',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.56',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#chapter.13',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section.9.2',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.298',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.298',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.304',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.313',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#subsection.14.2.1',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#subsection.14.3.2',\n",
       "  ''],\n",
       " ['',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '',\n",
       "  '168 A The VMLS package',\n",
       "  '',\n",
       "  'D[\"setosa\"] a 50× 4 matrix with 50 examples of Iris Setosa',\n",
       "  'D[\"versicolor\"]: a 50× 4 matrix with 50 examples of Iris Versicolor',\n",
       "  'D[\"virginica\"]: a 50× 4 matrix with 50 examples of Iris Virginica.',\n",
       "  '',\n",
       "  'The columns give values for four features: sepal length in cm, sepal width in',\n",
       "  'cm, petal length in cm, petal width in cm.',\n",
       "  '',\n",
       "  'ozone_data(). Returns a vector of length 336 = 14 · 24 with the hourly ozone',\n",
       "  'levels at Azusa, California, during the first 14 days of July 2014 (discussed',\n",
       "  'on page 319 in VMLS).',\n",
       "  '',\n",
       "  'regularized_fit_data(). Returns a dictionary D with data for the regularized',\n",
       "  'data fitting example on page 329 of VMLS. The items in the dictionary are:',\n",
       "  '',\n",
       "  'D[\"xtrain\"]: vector of length 10',\n",
       "  'D[\"ytrain\"]: vector of length 10',\n",
       "  'D[\"xtest\"]: vector of length 20',\n",
       "  'D[\"ytest\"]: vector of length 20.',\n",
       "  '',\n",
       "  'portfolio_data(). Returns a tuple (R, Rtest) with data for the portfolio op-',\n",
       "  'timization example in section 17.1.3 of VMLS. R is a 2000× 20 matrix with',\n",
       "  'daily returns over a period of 2000 days. The first 19 columns are returns for',\n",
       "  '19 stocks; the last column is for a risk-free asset. Rtest is a 500× 20 matrix',\n",
       "  'with daily returns over a different period of 500 days.',\n",
       "  '',\n",
       "  'lq_estimation_data(). Returns a 2 × 100 matrix with the measurement data',\n",
       "  'for the linear quadratic state estimation example of section 17.3.1 of VMLS.',\n",
       "  '',\n",
       "  'orth_dist_reg_data() Returns a tuple (xd, yd) with the data for the orthog-',\n",
       "  'onal distance regression example on page 400 in VMLS.',\n",
       "  '',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.358',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#figure.15.11',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#subsection.17.1.3',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#subsection.17.3.1',\n",
       "  'https://web.stanford.edu/\\\\%7Eboyd/vmls/vmls.pdf#section*.417',\n",
       "  '']]"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "megalist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " '',\n",
       " 'vi Contents',\n",
       " '',\n",
       " '']"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "megalist[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "getpage = lambda lis,num: lis[num-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Empty'"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "getpage(megalist,2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'list' object has no attribute 'T'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-99-e91a7598fa15>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmegalist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m: 'list' object has no attribute 'T'"
     ]
    }
   ],
   "source": [
    "pd.DataFrame(megalist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "newlist = np.array(megalist).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>[, , , , , , , , , , , , , , , , , , , , , , ,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Empty</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>[, , , , , , , , , , , , , , , , , , , , , , ,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>[, , , , , , , , , , , , , , , , , , , , , , ,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>[, , , , , , , , , , , , , , , , , , , , , , ,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>173</td>\n",
       "      <td>Empty</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>174</td>\n",
       "      <td>[, , , , , , , , , , , , , , , , , , , , , , ,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>175</td>\n",
       "      <td>[, , , , , , , , , , , , , , , , , , , , , , ,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>176</td>\n",
       "      <td>[, , , , , , , , , , , , , , , , , , , , , , ,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>177</td>\n",
       "      <td>[, , , , , , , , , , , , , , , , , , , , , , ,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>178 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     0\n",
       "0    [, , , , , , , , , , , , , , , , , , , , , , ,...\n",
       "1                                                Empty\n",
       "2    [, , , , , , , , , , , , , , , , , , , , , , ,...\n",
       "3    [, , , , , , , , , , , , , , , , , , , , , , ,...\n",
       "4    [, , , , , , , , , , , , , , , , , , , , , , ,...\n",
       "..                                                 ...\n",
       "173                                              Empty\n",
       "174  [, , , , , , , , , , , , , , , , , , , , , , ,...\n",
       "175  [, , , , , , , , , , , , , , , , , , , , , , ,...\n",
       "176  [, , , , , , , , , , , , , , , , , , , , , , ,...\n",
       "177  [, , , , , , , , , , , , , , , , , , , , , , ,...\n",
       "\n",
       "[178 rows x 1 columns]"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(newlist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
